1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/prefetch.h>
34#include <linux/ip.h>
35#include <linux/ipv6.h>
36#include <linux/tcp.h>
37#include <linux/indirect_call_wrapper.h>
38#include <net/ip6_checksum.h>
39#include <net/page_pool.h>
40#include <net/inet_ecn.h>
41#include "en.h"
42#include "en_tc.h"
43#include "eswitch.h"
44#include "en_rep.h"
45#include "en/rep/tc.h"
46#include "ipoib/ipoib.h"
47#include "en_accel/ipsec_rxtx.h"
48#include "en_accel/tls_rxtx.h"
49#include "lib/clock.h"
50#include "en/xdp.h"
51#include "en/xsk/rx.h"
52#include "en/health.h"
53
54static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
55{
56 return config->rx_filter == HWTSTAMP_FILTER_ALL;
57}
58
59static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq,
60 u32 cqcc, void *data)
61{
62 u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
63
64 memcpy(data, mlx5_cqwq_get_wqe(wq, ci), sizeof(struct mlx5_cqe64));
65}
66
67static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
68 struct mlx5_cqwq *wq,
69 u32 cqcc)
70{
71 struct mlx5e_cq_decomp *cqd = &rq->cqd;
72 struct mlx5_cqe64 *title = &cqd->title;
73
74 mlx5e_read_cqe_slot(wq, cqcc, title);
75 cqd->left = be32_to_cpu(title->byte_cnt);
76 cqd->wqe_counter = be16_to_cpu(title->wqe_counter);
77 rq->stats->cqe_compress_blks++;
78}
79
80static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq *wq,
81 struct mlx5e_cq_decomp *cqd,
82 u32 cqcc)
83{
84 mlx5e_read_cqe_slot(wq, cqcc, cqd->mini_arr);
85 cqd->mini_arr_idx = 0;
86}
87
88static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq *wq, int n)
89{
90 u32 cqcc = wq->cc;
91 u8 op_own = mlx5_cqwq_get_ctr_wrap_cnt(wq, cqcc) & 1;
92 u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
93 u32 wq_sz = mlx5_cqwq_get_size(wq);
94 u32 ci_top = min_t(u32, wq_sz, ci + n);
95
96 for (; ci < ci_top; ci++, n--) {
97 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
98
99 cqe->op_own = op_own;
100 }
101
102 if (unlikely(ci == wq_sz)) {
103 op_own = !op_own;
104 for (ci = 0; ci < n; ci++) {
105 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
106
107 cqe->op_own = op_own;
108 }
109 }
110}
111
112static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
113 struct mlx5_cqwq *wq,
114 u32 cqcc)
115{
116 struct mlx5e_cq_decomp *cqd = &rq->cqd;
117 struct mlx5_mini_cqe8 *mini_cqe = &cqd->mini_arr[cqd->mini_arr_idx];
118 struct mlx5_cqe64 *title = &cqd->title;
119
120 title->byte_cnt = mini_cqe->byte_cnt;
121 title->check_sum = mini_cqe->checksum;
122 title->op_own &= 0xf0;
123 title->op_own |= 0x01 & (cqcc >> wq->fbc.log_sz);
124 title->wqe_counter = cpu_to_be16(cqd->wqe_counter);
125
126 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
127 cqd->wqe_counter += mpwrq_get_cqe_consumed_strides(title);
128 else
129 cqd->wqe_counter =
130 mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, cqd->wqe_counter + 1);
131}
132
133static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
134 struct mlx5_cqwq *wq,
135 u32 cqcc)
136{
137 struct mlx5e_cq_decomp *cqd = &rq->cqd;
138
139 mlx5e_decompress_cqe(rq, wq, cqcc);
140 cqd->title.rss_hash_type = 0;
141 cqd->title.rss_hash_result = 0;
142}
143
144static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
145 struct mlx5_cqwq *wq,
146 int update_owner_only,
147 int budget_rem)
148{
149 struct mlx5e_cq_decomp *cqd = &rq->cqd;
150 u32 cqcc = wq->cc + update_owner_only;
151 u32 cqe_count;
152 u32 i;
153
154 cqe_count = min_t(u32, cqd->left, budget_rem);
155
156 for (i = update_owner_only; i < cqe_count;
157 i++, cqd->mini_arr_idx++, cqcc++) {
158 if (cqd->mini_arr_idx == MLX5_MINI_CQE_ARRAY_SIZE)
159 mlx5e_read_mini_arr_slot(wq, cqd, cqcc);
160
161 mlx5e_decompress_cqe_no_hash(rq, wq, cqcc);
162 INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
163 mlx5e_handle_rx_cqe, rq, &cqd->title);
164 }
165 mlx5e_cqes_update_owner(wq, cqcc - wq->cc);
166 wq->cc = cqcc;
167 cqd->left -= cqe_count;
168 rq->stats->cqe_compress_pkts += cqe_count;
169
170 return cqe_count;
171}
172
173static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
174 struct mlx5_cqwq *wq,
175 int budget_rem)
176{
177 struct mlx5e_cq_decomp *cqd = &rq->cqd;
178 u32 cc = wq->cc;
179
180 mlx5e_read_title_slot(rq, wq, cc);
181 mlx5e_read_mini_arr_slot(wq, cqd, cc + 1);
182 mlx5e_decompress_cqe(rq, wq, cc);
183 INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
184 mlx5e_handle_rx_cqe, rq, &cqd->title);
185 cqd->mini_arr_idx++;
186
187 return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1;
188}
189
190static inline bool mlx5e_page_is_reserved(struct page *page)
191{
192 return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id();
193}
194
195static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
196 struct mlx5e_dma_info *dma_info)
197{
198 struct mlx5e_page_cache *cache = &rq->page_cache;
199 u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);
200 struct mlx5e_rq_stats *stats = rq->stats;
201
202 if (tail_next == cache->head) {
203 stats->cache_full++;
204 return false;
205 }
206
207 if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
208 stats->cache_waive++;
209 return false;
210 }
211
212 cache->page_cache[cache->tail] = *dma_info;
213 cache->tail = tail_next;
214 return true;
215}
216
217static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
218 struct mlx5e_dma_info *dma_info)
219{
220 struct mlx5e_page_cache *cache = &rq->page_cache;
221 struct mlx5e_rq_stats *stats = rq->stats;
222
223 if (unlikely(cache->head == cache->tail)) {
224 stats->cache_empty++;
225 return false;
226 }
227
228 if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
229 stats->cache_busy++;
230 return false;
231 }
232
233 *dma_info = cache->page_cache[cache->head];
234 cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
235 stats->cache_reuse++;
236
237 dma_sync_single_for_device(rq->pdev, dma_info->addr,
238 PAGE_SIZE,
239 DMA_FROM_DEVICE);
240 return true;
241}
242
243static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
244 struct mlx5e_dma_info *dma_info)
245{
246 if (mlx5e_rx_cache_get(rq, dma_info))
247 return 0;
248
249 dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
250 if (unlikely(!dma_info->page))
251 return -ENOMEM;
252
253 dma_info->addr = dma_map_page(rq->pdev, dma_info->page, 0,
254 PAGE_SIZE, rq->buff.map_dir);
255 if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
256 page_pool_recycle_direct(rq->page_pool, dma_info->page);
257 dma_info->page = NULL;
258 return -ENOMEM;
259 }
260
261 return 0;
262}
263
264static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
265 struct mlx5e_dma_info *dma_info)
266{
267 if (rq->umem)
268 return mlx5e_xsk_page_alloc_umem(rq, dma_info);
269 else
270 return mlx5e_page_alloc_pool(rq, dma_info);
271}
272
273void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
274{
275 dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
276}
277
278void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
279 struct mlx5e_dma_info *dma_info,
280 bool recycle)
281{
282 if (likely(recycle)) {
283 if (mlx5e_rx_cache_put(rq, dma_info))
284 return;
285
286 mlx5e_page_dma_unmap(rq, dma_info);
287 page_pool_recycle_direct(rq->page_pool, dma_info->page);
288 } else {
289 mlx5e_page_dma_unmap(rq, dma_info);
290 page_pool_release_page(rq->page_pool, dma_info->page);
291 put_page(dma_info->page);
292 }
293}
294
295static inline void mlx5e_page_release(struct mlx5e_rq *rq,
296 struct mlx5e_dma_info *dma_info,
297 bool recycle)
298{
299 if (rq->umem)
300
301
302
303
304 xsk_buff_free(dma_info->xsk);
305 else
306 mlx5e_page_release_dynamic(rq, dma_info, recycle);
307}
308
309static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
310 struct mlx5e_wqe_frag_info *frag)
311{
312 int err = 0;
313
314 if (!frag->offset)
315
316
317
318
319
320 err = mlx5e_page_alloc(rq, frag->di);
321
322 return err;
323}
324
325static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
326 struct mlx5e_wqe_frag_info *frag,
327 bool recycle)
328{
329 if (frag->last_in_page)
330 mlx5e_page_release(rq, frag->di, recycle);
331}
332
333static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
334{
335 return &rq->wqe.frags[ix << rq->wqe.info.log_num_frags];
336}
337
338static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
339 u16 ix)
340{
341 struct mlx5e_wqe_frag_info *frag = get_frag(rq, ix);
342 int err;
343 int i;
344
345 for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) {
346 err = mlx5e_get_rx_frag(rq, frag);
347 if (unlikely(err))
348 goto free_frags;
349
350 wqe->data[i].addr = cpu_to_be64(frag->di->addr +
351 frag->offset + rq->buff.headroom);
352 }
353
354 return 0;
355
356free_frags:
357 while (--i >= 0)
358 mlx5e_put_rx_frag(rq, --frag, true);
359
360 return err;
361}
362
363static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
364 struct mlx5e_wqe_frag_info *wi,
365 bool recycle)
366{
367 int i;
368
369 for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
370 mlx5e_put_rx_frag(rq, wi, recycle);
371}
372
373void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
374{
375 struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
376
377 mlx5e_free_rx_wqe(rq, wi, false);
378}
379
380static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
381{
382 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
383 int err;
384 int i;
385
386 if (rq->umem) {
387 int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
388
389
390
391
392
393 if (unlikely(!xsk_buff_can_alloc(rq->umem, pages_desired)))
394 return -ENOMEM;
395 }
396
397 for (i = 0; i < wqe_bulk; i++) {
398 struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
399
400 err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i);
401 if (unlikely(err))
402 goto free_wqes;
403 }
404
405 return 0;
406
407free_wqes:
408 while (--i >= 0)
409 mlx5e_dealloc_rx_wqe(rq, ix + i);
410
411 return err;
412}
413
414static inline void
415mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
416 struct mlx5e_dma_info *di, u32 frag_offset, u32 len,
417 unsigned int truesize)
418{
419 dma_sync_single_for_cpu(rq->pdev,
420 di->addr + frag_offset,
421 len, DMA_FROM_DEVICE);
422 page_ref_inc(di->page);
423 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
424 di->page, frag_offset, len, truesize);
425}
426
427static inline void
428mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
429 struct mlx5e_dma_info *dma_info,
430 int offset_from, u32 headlen)
431{
432 const void *from = page_address(dma_info->page) + offset_from;
433
434 unsigned int len = ALIGN(headlen, sizeof(long));
435
436 dma_sync_single_for_cpu(pdev, dma_info->addr + offset_from, len,
437 DMA_FROM_DEVICE);
438 skb_copy_to_linear_data(skb, from, len);
439}
440
441static void
442mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
443{
444 bool no_xdp_xmit;
445 struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
446 int i;
447
448
449 if (bitmap_full(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE))
450 return;
451
452 no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap,
453 MLX5_MPWRQ_PAGES_PER_WQE);
454
455 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
456 if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
457 mlx5e_page_release(rq, &dma_info[i], recycle);
458}
459
460static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
461{
462 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
463
464 do {
465 u16 next_wqe_index = mlx5_wq_ll_get_wqe_next_ix(wq, wq->head);
466
467 mlx5_wq_ll_push(wq, next_wqe_index);
468 } while (--n);
469
470
471 dma_wmb();
472
473 mlx5_wq_ll_update_db_record(wq);
474}
475
476static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
477{
478 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
479 struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
480 struct mlx5e_icosq *sq = &rq->channel->icosq;
481 struct mlx5_wq_cyc *wq = &sq->wq;
482 struct mlx5e_umr_wqe *umr_wqe;
483 u16 xlt_offset = ix << (MLX5E_LOG_ALIGNED_MPWQE_PPW - 1);
484 u16 pi;
485 int err;
486 int i;
487
488
489
490
491 if (rq->umem &&
492 unlikely(!xsk_buff_can_alloc(rq->umem, MLX5_MPWRQ_PAGES_PER_WQE))) {
493 err = -ENOMEM;
494 goto err;
495 }
496
497 pi = mlx5e_icosq_get_next_pi(sq, MLX5E_UMR_WQEBBS);
498 umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
499 memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts));
500
501 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
502 err = mlx5e_page_alloc(rq, dma_info);
503 if (unlikely(err))
504 goto err_unmap;
505 umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
506 }
507
508 bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
509 wi->consumed_strides = 0;
510
511 umr_wqe->ctrl.opmod_idx_opcode =
512 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
513 MLX5_OPCODE_UMR);
514 umr_wqe->uctrl.xlt_offset = cpu_to_be16(xlt_offset);
515
516 sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
517 .wqe_type = MLX5E_ICOSQ_WQE_UMR_RX,
518 .num_wqebbs = MLX5E_UMR_WQEBBS,
519 .umr.rq = rq,
520 };
521
522 sq->pc += MLX5E_UMR_WQEBBS;
523
524 sq->doorbell_cseg = &umr_wqe->ctrl;
525
526 return 0;
527
528err_unmap:
529 while (--i >= 0) {
530 dma_info--;
531 mlx5e_page_release(rq, dma_info, true);
532 }
533
534err:
535 rq->stats->buff_alloc_err++;
536
537 return err;
538}
539
540void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
541{
542 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
543
544 mlx5e_free_rx_mpwqe(rq, wi, false);
545}
546
547bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
548{
549 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
550 u8 wqe_bulk;
551 int err;
552
553 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
554 return false;
555
556 wqe_bulk = rq->wqe.info.wqe_bulk;
557
558 if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
559 return false;
560
561 do {
562 u16 head = mlx5_wq_cyc_get_head(wq);
563
564 err = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
565 if (unlikely(err)) {
566 rq->stats->buff_alloc_err++;
567 break;
568 }
569
570 mlx5_wq_cyc_push_n(wq, wqe_bulk);
571 } while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);
572
573
574 dma_wmb();
575
576 mlx5_wq_cyc_update_db_record(wq);
577
578 return !!err;
579}
580
581int mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
582{
583 struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
584 struct mlx5_cqe64 *cqe;
585 u16 sqcc;
586 int i;
587
588 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
589 return 0;
590
591 cqe = mlx5_cqwq_get_cqe(&cq->wq);
592 if (likely(!cqe))
593 return 0;
594
595
596
597
598 sqcc = sq->cc;
599
600 i = 0;
601 do {
602 u16 wqe_counter;
603 bool last_wqe;
604
605 mlx5_cqwq_pop(&cq->wq);
606
607 wqe_counter = be16_to_cpu(cqe->wqe_counter);
608
609 do {
610 struct mlx5e_icosq_wqe_info *wi;
611 u16 ci;
612
613 last_wqe = (sqcc == wqe_counter);
614
615 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
616 wi = &sq->db.wqe_info[ci];
617 sqcc += wi->num_wqebbs;
618
619 if (last_wqe && unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
620 netdev_WARN_ONCE(cq->channel->netdev,
621 "Bad OP in ICOSQ CQE: 0x%x\n",
622 get_cqe_opcode(cqe));
623 mlx5e_dump_error_cqe(&sq->cq, sq->sqn,
624 (struct mlx5_err_cqe *)cqe);
625 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state))
626 queue_work(cq->channel->priv->wq, &sq->recover_work);
627 break;
628 }
629
630 switch (wi->wqe_type) {
631 case MLX5E_ICOSQ_WQE_UMR_RX:
632 wi->umr.rq->mpwqe.umr_completed++;
633 break;
634 case MLX5E_ICOSQ_WQE_NOP:
635 break;
636 default:
637 netdev_WARN_ONCE(cq->channel->netdev,
638 "Bad WQE type in ICOSQ WQE info: 0x%x\n",
639 wi->wqe_type);
640 }
641 } while (!last_wqe);
642 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
643
644 sq->cc = sqcc;
645
646 mlx5_cqwq_update_db_record(&cq->wq);
647
648 return i;
649}
650
651bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
652{
653 struct mlx5e_icosq *sq = &rq->channel->icosq;
654 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
655 u8 umr_completed = rq->mpwqe.umr_completed;
656 int alloc_err = 0;
657 u8 missing, i;
658 u16 head;
659
660 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
661 return false;
662
663 if (umr_completed) {
664 mlx5e_post_rx_mpwqe(rq, umr_completed);
665 rq->mpwqe.umr_in_progress -= umr_completed;
666 rq->mpwqe.umr_completed = 0;
667 }
668
669 missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress;
670
671 if (unlikely(rq->mpwqe.umr_in_progress > rq->mpwqe.umr_last_bulk))
672 rq->stats->congst_umr++;
673
674#define UMR_WQE_BULK (2)
675 if (likely(missing < UMR_WQE_BULK))
676 return false;
677
678 head = rq->mpwqe.actual_wq_head;
679 i = missing;
680 do {
681 alloc_err = mlx5e_alloc_rx_mpwqe(rq, head);
682
683 if (unlikely(alloc_err))
684 break;
685 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
686 } while (--i);
687
688 rq->mpwqe.umr_last_bulk = missing - i;
689 if (sq->doorbell_cseg) {
690 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
691 sq->doorbell_cseg = NULL;
692 }
693
694 rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
695 rq->mpwqe.actual_wq_head = head;
696
697
698
699
700
701
702
703 if (unlikely(alloc_err == -ENOMEM && rq->umem))
704 return true;
705
706 return false;
707}
708
709static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp)
710{
711 u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
712 u8 tcp_ack = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
713 (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
714
715 tcp->check = 0;
716 tcp->psh = get_cqe_lro_tcppsh(cqe);
717
718 if (tcp_ack) {
719 tcp->ack = 1;
720 tcp->ack_seq = cqe->lro_ack_seq_num;
721 tcp->window = cqe->lro_tcp_win;
722 }
723}
724
725static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
726 u32 cqe_bcnt)
727{
728 struct ethhdr *eth = (struct ethhdr *)(skb->data);
729 struct tcphdr *tcp;
730 int network_depth = 0;
731 __wsum check;
732 __be16 proto;
733 u16 tot_len;
734 void *ip_p;
735
736 proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
737
738 tot_len = cqe_bcnt - network_depth;
739 ip_p = skb->data + network_depth;
740
741 if (proto == htons(ETH_P_IP)) {
742 struct iphdr *ipv4 = ip_p;
743
744 tcp = ip_p + sizeof(struct iphdr);
745 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
746
747 ipv4->ttl = cqe->lro_min_ttl;
748 ipv4->tot_len = cpu_to_be16(tot_len);
749 ipv4->check = 0;
750 ipv4->check = ip_fast_csum((unsigned char *)ipv4,
751 ipv4->ihl);
752
753 mlx5e_lro_update_tcp_hdr(cqe, tcp);
754 check = csum_partial(tcp, tcp->doff * 4,
755 csum_unfold((__force __sum16)cqe->check_sum));
756
757 tcp->check = csum_tcpudp_magic(ipv4->saddr, ipv4->daddr,
758 tot_len - sizeof(struct iphdr),
759 IPPROTO_TCP, check);
760 } else {
761 u16 payload_len = tot_len - sizeof(struct ipv6hdr);
762 struct ipv6hdr *ipv6 = ip_p;
763
764 tcp = ip_p + sizeof(struct ipv6hdr);
765 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
766
767 ipv6->hop_limit = cqe->lro_min_ttl;
768 ipv6->payload_len = cpu_to_be16(payload_len);
769
770 mlx5e_lro_update_tcp_hdr(cqe, tcp);
771 check = csum_partial(tcp, tcp->doff * 4,
772 csum_unfold((__force __sum16)cqe->check_sum));
773
774 tcp->check = csum_ipv6_magic(&ipv6->saddr, &ipv6->daddr, payload_len,
775 IPPROTO_TCP, check);
776 }
777}
778
779static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
780 struct sk_buff *skb)
781{
782 u8 cht = cqe->rss_hash_type;
783 int ht = (cht & CQE_RSS_HTYPE_L4) ? PKT_HASH_TYPE_L4 :
784 (cht & CQE_RSS_HTYPE_IP) ? PKT_HASH_TYPE_L3 :
785 PKT_HASH_TYPE_NONE;
786 skb_set_hash(skb, be32_to_cpu(cqe->rss_hash_result), ht);
787}
788
789static inline bool is_last_ethertype_ip(struct sk_buff *skb, int *network_depth,
790 __be16 *proto)
791{
792 *proto = ((struct ethhdr *)skb->data)->h_proto;
793 *proto = __vlan_get_protocol(skb, *proto, network_depth);
794
795 if (*proto == htons(ETH_P_IP))
796 return pskb_may_pull(skb, *network_depth + sizeof(struct iphdr));
797
798 if (*proto == htons(ETH_P_IPV6))
799 return pskb_may_pull(skb, *network_depth + sizeof(struct ipv6hdr));
800
801 return false;
802}
803
804static inline void mlx5e_enable_ecn(struct mlx5e_rq *rq, struct sk_buff *skb)
805{
806 int network_depth = 0;
807 __be16 proto;
808 void *ip;
809 int rc;
810
811 if (unlikely(!is_last_ethertype_ip(skb, &network_depth, &proto)))
812 return;
813
814 ip = skb->data + network_depth;
815 rc = ((proto == htons(ETH_P_IP)) ? IP_ECN_set_ce((struct iphdr *)ip) :
816 IP6_ECN_set_ce(skb, (struct ipv6hdr *)ip));
817
818 rq->stats->ecn_mark += !!rc;
819}
820
821static u8 get_ip_proto(struct sk_buff *skb, int network_depth, __be16 proto)
822{
823 void *ip_p = skb->data + network_depth;
824
825 return (proto == htons(ETH_P_IP)) ? ((struct iphdr *)ip_p)->protocol :
826 ((struct ipv6hdr *)ip_p)->nexthdr;
827}
828
829#define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
830
831#define MAX_PADDING 8
832
833static void
834tail_padding_csum_slow(struct sk_buff *skb, int offset, int len,
835 struct mlx5e_rq_stats *stats)
836{
837 stats->csum_complete_tail_slow++;
838 skb->csum = csum_block_add(skb->csum,
839 skb_checksum(skb, offset, len, 0),
840 offset);
841}
842
843static void
844tail_padding_csum(struct sk_buff *skb, int offset,
845 struct mlx5e_rq_stats *stats)
846{
847 u8 tail_padding[MAX_PADDING];
848 int len = skb->len - offset;
849 void *tail;
850
851 if (unlikely(len > MAX_PADDING)) {
852 tail_padding_csum_slow(skb, offset, len, stats);
853 return;
854 }
855
856 tail = skb_header_pointer(skb, offset, len, tail_padding);
857 if (unlikely(!tail)) {
858 tail_padding_csum_slow(skb, offset, len, stats);
859 return;
860 }
861
862 stats->csum_complete_tail++;
863 skb->csum = csum_block_add(skb->csum, csum_partial(tail, len, 0), offset);
864}
865
866static void
867mlx5e_skb_csum_fixup(struct sk_buff *skb, int network_depth, __be16 proto,
868 struct mlx5e_rq_stats *stats)
869{
870 struct ipv6hdr *ip6;
871 struct iphdr *ip4;
872 int pkt_len;
873
874
875 if (network_depth > ETH_HLEN)
876
877
878
879
880 skb->csum = csum_partial(skb->data + ETH_HLEN,
881 network_depth - ETH_HLEN,
882 skb->csum);
883
884
885 switch (proto) {
886 case htons(ETH_P_IP):
887 ip4 = (struct iphdr *)(skb->data + network_depth);
888 pkt_len = network_depth + ntohs(ip4->tot_len);
889 break;
890 case htons(ETH_P_IPV6):
891 ip6 = (struct ipv6hdr *)(skb->data + network_depth);
892 pkt_len = network_depth + sizeof(*ip6) + ntohs(ip6->payload_len);
893 break;
894 default:
895 return;
896 }
897
898 if (likely(pkt_len >= skb->len))
899 return;
900
901 tail_padding_csum(skb, pkt_len, stats);
902}
903
904static inline void mlx5e_handle_csum(struct net_device *netdev,
905 struct mlx5_cqe64 *cqe,
906 struct mlx5e_rq *rq,
907 struct sk_buff *skb,
908 bool lro)
909{
910 struct mlx5e_rq_stats *stats = rq->stats;
911 int network_depth = 0;
912 __be16 proto;
913
914 if (unlikely(!(netdev->features & NETIF_F_RXCSUM)))
915 goto csum_none;
916
917 if (lro) {
918 skb->ip_summed = CHECKSUM_UNNECESSARY;
919 stats->csum_unnecessary++;
920 return;
921 }
922
923
924 if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state))
925 goto csum_unnecessary;
926
927
928
929
930
931
932
933
934
935 if (short_frame(skb->len))
936 goto csum_unnecessary;
937
938 if (likely(is_last_ethertype_ip(skb, &network_depth, &proto))) {
939 if (unlikely(get_ip_proto(skb, network_depth, proto) == IPPROTO_SCTP))
940 goto csum_unnecessary;
941
942 stats->csum_complete++;
943 skb->ip_summed = CHECKSUM_COMPLETE;
944 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
945
946 if (test_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state))
947 return;
948
949
950 mlx5e_skb_csum_fixup(skb, network_depth, proto, stats);
951 return;
952 }
953
954csum_unnecessary:
955 if (likely((cqe->hds_ip_ext & CQE_L3_OK) &&
956 (cqe->hds_ip_ext & CQE_L4_OK))) {
957 skb->ip_summed = CHECKSUM_UNNECESSARY;
958 if (cqe_is_tunneled(cqe)) {
959 skb->csum_level = 1;
960 skb->encapsulation = 1;
961 stats->csum_unnecessary_inner++;
962 return;
963 }
964 stats->csum_unnecessary++;
965 return;
966 }
967csum_none:
968 skb->ip_summed = CHECKSUM_NONE;
969 stats->csum_none++;
970}
971
972#define MLX5E_CE_BIT_MASK 0x80
973
974static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
975 u32 cqe_bcnt,
976 struct mlx5e_rq *rq,
977 struct sk_buff *skb)
978{
979 u8 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
980 struct mlx5e_rq_stats *stats = rq->stats;
981 struct net_device *netdev = rq->netdev;
982
983 skb->mac_len = ETH_HLEN;
984
985#ifdef CONFIG_MLX5_EN_TLS
986 mlx5e_tls_handle_rx_skb(netdev, skb, &cqe_bcnt);
987#endif
988
989 if (lro_num_seg > 1) {
990 mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
991 skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
992
993
994
995 stats->packets += lro_num_seg - 1;
996 stats->lro_packets++;
997 stats->lro_bytes += cqe_bcnt;
998 }
999
1000 if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
1001 skb_hwtstamps(skb)->hwtstamp =
1002 mlx5_timecounter_cyc2time(rq->clock, get_cqe_ts(cqe));
1003
1004 skb_record_rx_queue(skb, rq->ix);
1005
1006 if (likely(netdev->features & NETIF_F_RXHASH))
1007 mlx5e_skb_set_hash(cqe, skb);
1008
1009 if (cqe_has_vlan(cqe)) {
1010 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1011 be16_to_cpu(cqe->vlan_info));
1012 stats->removed_vlan_packets++;
1013 }
1014
1015 skb->mark = be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK;
1016
1017 mlx5e_handle_csum(netdev, cqe, rq, skb, !!lro_num_seg);
1018
1019 if (unlikely(cqe->ml_path & MLX5E_CE_BIT_MASK))
1020 mlx5e_enable_ecn(rq, skb);
1021
1022 skb->protocol = eth_type_trans(skb, netdev);
1023}
1024
1025static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
1026 struct mlx5_cqe64 *cqe,
1027 u32 cqe_bcnt,
1028 struct sk_buff *skb)
1029{
1030 struct mlx5e_rq_stats *stats = rq->stats;
1031
1032 stats->packets++;
1033 stats->bytes += cqe_bcnt;
1034 mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
1035}
1036
1037static inline
1038struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
1039 u32 frag_size, u16 headroom,
1040 u32 cqe_bcnt)
1041{
1042 struct sk_buff *skb = build_skb(va, frag_size);
1043
1044 if (unlikely(!skb)) {
1045 rq->stats->buff_alloc_err++;
1046 return NULL;
1047 }
1048
1049 skb_reserve(skb, headroom);
1050 skb_put(skb, cqe_bcnt);
1051
1052 return skb;
1053}
1054
1055static void mlx5e_fill_xdp_buff(struct mlx5e_rq *rq, void *va, u16 headroom,
1056 u32 len, struct xdp_buff *xdp)
1057{
1058 xdp->data_hard_start = va;
1059 xdp->data = va + headroom;
1060 xdp_set_data_meta_invalid(xdp);
1061 xdp->data_end = xdp->data + len;
1062 xdp->rxq = &rq->xdp_rxq;
1063 xdp->frame_sz = rq->buff.frame0_sz;
1064}
1065
1066struct sk_buff *
1067mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1068 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
1069{
1070 struct mlx5e_dma_info *di = wi->di;
1071 u16 rx_headroom = rq->buff.headroom;
1072 struct xdp_buff xdp;
1073 struct sk_buff *skb;
1074 void *va, *data;
1075 bool consumed;
1076 u32 frag_size;
1077
1078 va = page_address(di->page) + wi->offset;
1079 data = va + rx_headroom;
1080 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
1081
1082 dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
1083 frag_size, DMA_FROM_DEVICE);
1084 prefetchw(va);
1085 prefetch(data);
1086
1087 rcu_read_lock();
1088 mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
1089 consumed = mlx5e_xdp_handle(rq, di, &cqe_bcnt, &xdp);
1090 rcu_read_unlock();
1091 if (consumed)
1092 return NULL;
1093
1094 rx_headroom = xdp.data - xdp.data_hard_start;
1095 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
1096 skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt);
1097 if (unlikely(!skb))
1098 return NULL;
1099
1100
1101 page_ref_inc(di->page);
1102
1103 return skb;
1104}
1105
1106struct sk_buff *
1107mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
1108 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
1109{
1110 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
1111 struct mlx5e_wqe_frag_info *head_wi = wi;
1112 u16 headlen = min_t(u32, MLX5E_RX_MAX_HEAD, cqe_bcnt);
1113 u16 frag_headlen = headlen;
1114 u16 byte_cnt = cqe_bcnt - headlen;
1115 struct sk_buff *skb;
1116
1117
1118
1119
1120 skb = napi_alloc_skb(rq->cq.napi,
1121 ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
1122 if (unlikely(!skb)) {
1123 rq->stats->buff_alloc_err++;
1124 return NULL;
1125 }
1126
1127 prefetchw(skb->data);
1128
1129 while (byte_cnt) {
1130 u16 frag_consumed_bytes =
1131 min_t(u16, frag_info->frag_size - frag_headlen, byte_cnt);
1132
1133 mlx5e_add_skb_frag(rq, skb, wi->di, wi->offset + frag_headlen,
1134 frag_consumed_bytes, frag_info->frag_stride);
1135 byte_cnt -= frag_consumed_bytes;
1136 frag_headlen = 0;
1137 frag_info++;
1138 wi++;
1139 }
1140
1141
1142 mlx5e_copy_skb_header(rq->pdev, skb, head_wi->di, head_wi->offset, headlen);
1143
1144 skb->tail += headlen;
1145 skb->len += headlen;
1146
1147 return skb;
1148}
1149
1150static void trigger_report(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1151{
1152 struct mlx5_err_cqe *err_cqe = (struct mlx5_err_cqe *)cqe;
1153
1154 if (cqe_syndrome_needs_recover(err_cqe->syndrome) &&
1155 !test_and_set_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state))
1156 queue_work(rq->channel->priv->wq, &rq->recover_work);
1157}
1158
1159void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1160{
1161 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1162 struct mlx5e_wqe_frag_info *wi;
1163 struct sk_buff *skb;
1164 u32 cqe_bcnt;
1165 u16 ci;
1166
1167 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1168 wi = get_frag(rq, ci);
1169 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1170
1171 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1172 trigger_report(rq, cqe);
1173 rq->stats->wqe_err++;
1174 goto free_wqe;
1175 }
1176
1177 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1178 mlx5e_skb_from_cqe_linear,
1179 mlx5e_skb_from_cqe_nonlinear,
1180 rq, cqe, wi, cqe_bcnt);
1181 if (!skb) {
1182
1183 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
1184
1185
1186
1187 goto wq_cyc_pop;
1188 }
1189 goto free_wqe;
1190 }
1191
1192 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1193 napi_gro_receive(rq->cq.napi, skb);
1194
1195free_wqe:
1196 mlx5e_free_rx_wqe(rq, wi, true);
1197wq_cyc_pop:
1198 mlx5_wq_cyc_pop(wq);
1199}
1200
1201#ifdef CONFIG_MLX5_ESWITCH
1202void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1203{
1204 struct net_device *netdev = rq->netdev;
1205 struct mlx5e_priv *priv = netdev_priv(netdev);
1206 struct mlx5e_rep_priv *rpriv = priv->ppriv;
1207 struct mlx5_eswitch_rep *rep = rpriv->rep;
1208 struct mlx5e_tc_update_priv tc_priv = {};
1209 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1210 struct mlx5e_wqe_frag_info *wi;
1211 struct sk_buff *skb;
1212 u32 cqe_bcnt;
1213 u16 ci;
1214
1215 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1216 wi = get_frag(rq, ci);
1217 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1218
1219 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1220 rq->stats->wqe_err++;
1221 goto free_wqe;
1222 }
1223
1224 skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
1225 if (!skb) {
1226
1227 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
1228
1229
1230
1231 goto wq_cyc_pop;
1232 }
1233 goto free_wqe;
1234 }
1235
1236 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1237
1238 if (rep->vlan && skb_vlan_tag_present(skb))
1239 skb_vlan_pop(skb);
1240
1241 if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))
1242 goto free_wqe;
1243
1244 napi_gro_receive(rq->cq.napi, skb);
1245
1246 mlx5_rep_tc_post_napi_receive(&tc_priv);
1247
1248free_wqe:
1249 mlx5e_free_rx_wqe(rq, wi, true);
1250wq_cyc_pop:
1251 mlx5_wq_cyc_pop(wq);
1252}
1253
1254void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq,
1255 struct mlx5_cqe64 *cqe)
1256{
1257 u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
1258 u16 wqe_id = be16_to_cpu(cqe->wqe_id);
1259 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
1260 u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
1261 u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
1262 u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
1263 u32 page_idx = wqe_offset >> PAGE_SHIFT;
1264 struct mlx5e_tc_update_priv tc_priv = {};
1265 struct mlx5e_rx_wqe_ll *wqe;
1266 struct mlx5_wq_ll *wq;
1267 struct sk_buff *skb;
1268 u16 cqe_bcnt;
1269
1270 wi->consumed_strides += cstrides;
1271
1272 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1273 trigger_report(rq, cqe);
1274 rq->stats->wqe_err++;
1275 goto mpwrq_cqe_out;
1276 }
1277
1278 if (unlikely(mpwrq_is_filler_cqe(cqe))) {
1279 struct mlx5e_rq_stats *stats = rq->stats;
1280
1281 stats->mpwqe_filler_cqes++;
1282 stats->mpwqe_filler_strides += cstrides;
1283 goto mpwrq_cqe_out;
1284 }
1285
1286 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
1287
1288 skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
1289 mlx5e_skb_from_cqe_mpwrq_linear,
1290 mlx5e_skb_from_cqe_mpwrq_nonlinear,
1291 rq, wi, cqe_bcnt, head_offset, page_idx);
1292 if (!skb)
1293 goto mpwrq_cqe_out;
1294
1295 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1296
1297 if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))
1298 goto mpwrq_cqe_out;
1299
1300 napi_gro_receive(rq->cq.napi, skb);
1301
1302 mlx5_rep_tc_post_napi_receive(&tc_priv);
1303
1304mpwrq_cqe_out:
1305 if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1306 return;
1307
1308 wq = &rq->mpwqe.wq;
1309 wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1310 mlx5e_free_rx_mpwqe(rq, wi, true);
1311 mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1312}
1313#endif
1314
1315struct sk_buff *
1316mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1317 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1318{
1319 u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
1320 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1321 u32 frag_offset = head_offset + headlen;
1322 u32 byte_cnt = cqe_bcnt - headlen;
1323 struct mlx5e_dma_info *head_di = di;
1324 struct sk_buff *skb;
1325
1326 skb = napi_alloc_skb(rq->cq.napi,
1327 ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
1328 if (unlikely(!skb)) {
1329 rq->stats->buff_alloc_err++;
1330 return NULL;
1331 }
1332
1333 prefetchw(skb->data);
1334
1335 if (unlikely(frag_offset >= PAGE_SIZE)) {
1336 di++;
1337 frag_offset -= PAGE_SIZE;
1338 }
1339
1340 while (byte_cnt) {
1341 u32 pg_consumed_bytes =
1342 min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
1343 unsigned int truesize =
1344 ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
1345
1346 mlx5e_add_skb_frag(rq, skb, di, frag_offset,
1347 pg_consumed_bytes, truesize);
1348 byte_cnt -= pg_consumed_bytes;
1349 frag_offset = 0;
1350 di++;
1351 }
1352
1353 mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, headlen);
1354
1355 skb->tail += headlen;
1356 skb->len += headlen;
1357
1358 return skb;
1359}
1360
1361struct sk_buff *
1362mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
1363 u16 cqe_bcnt, u32 head_offset, u32 page_idx)
1364{
1365 struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
1366 u16 rx_headroom = rq->buff.headroom;
1367 u32 cqe_bcnt32 = cqe_bcnt;
1368 struct xdp_buff xdp;
1369 struct sk_buff *skb;
1370 void *va, *data;
1371 u32 frag_size;
1372 bool consumed;
1373
1374
1375 if (unlikely(cqe_bcnt > rq->hw_mtu)) {
1376 rq->stats->oversize_pkts_sw_drop++;
1377 return NULL;
1378 }
1379
1380 va = page_address(di->page) + head_offset;
1381 data = va + rx_headroom;
1382 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1383
1384 dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
1385 frag_size, DMA_FROM_DEVICE);
1386 prefetchw(va);
1387 prefetch(data);
1388
1389 rcu_read_lock();
1390 mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt32, &xdp);
1391 consumed = mlx5e_xdp_handle(rq, di, &cqe_bcnt32, &xdp);
1392 rcu_read_unlock();
1393 if (consumed) {
1394 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
1395 __set_bit(page_idx, wi->xdp_xmit_bitmap);
1396 return NULL;
1397 }
1398
1399 rx_headroom = xdp.data - xdp.data_hard_start;
1400 frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt32);
1401 skb = mlx5e_build_linear_skb(rq, va, frag_size, rx_headroom, cqe_bcnt32);
1402 if (unlikely(!skb))
1403 return NULL;
1404
1405
1406 page_ref_inc(di->page);
1407
1408 return skb;
1409}
1410
1411void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1412{
1413 u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
1414 u16 wqe_id = be16_to_cpu(cqe->wqe_id);
1415 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
1416 u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
1417 u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
1418 u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
1419 u32 page_idx = wqe_offset >> PAGE_SHIFT;
1420 struct mlx5e_rx_wqe_ll *wqe;
1421 struct mlx5_wq_ll *wq;
1422 struct sk_buff *skb;
1423 u16 cqe_bcnt;
1424
1425 wi->consumed_strides += cstrides;
1426
1427 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1428 trigger_report(rq, cqe);
1429 rq->stats->wqe_err++;
1430 goto mpwrq_cqe_out;
1431 }
1432
1433 if (unlikely(mpwrq_is_filler_cqe(cqe))) {
1434 struct mlx5e_rq_stats *stats = rq->stats;
1435
1436 stats->mpwqe_filler_cqes++;
1437 stats->mpwqe_filler_strides += cstrides;
1438 goto mpwrq_cqe_out;
1439 }
1440
1441 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
1442
1443 skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
1444 mlx5e_skb_from_cqe_mpwrq_linear,
1445 mlx5e_skb_from_cqe_mpwrq_nonlinear,
1446 rq, wi, cqe_bcnt, head_offset, page_idx);
1447 if (!skb)
1448 goto mpwrq_cqe_out;
1449
1450 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1451 napi_gro_receive(rq->cq.napi, skb);
1452
1453mpwrq_cqe_out:
1454 if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
1455 return;
1456
1457 wq = &rq->mpwqe.wq;
1458 wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1459 mlx5e_free_rx_mpwqe(rq, wi, true);
1460 mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1461}
1462
1463int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
1464{
1465 struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
1466 struct mlx5_cqwq *cqwq = &cq->wq;
1467 struct mlx5_cqe64 *cqe;
1468 int work_done = 0;
1469
1470 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
1471 return 0;
1472
1473 if (rq->page_pool)
1474 page_pool_nid_changed(rq->page_pool, numa_mem_id());
1475
1476 if (rq->cqd.left) {
1477 work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget);
1478 if (rq->cqd.left || work_done >= budget)
1479 goto out;
1480 }
1481
1482 cqe = mlx5_cqwq_get_cqe(cqwq);
1483 if (!cqe) {
1484 if (unlikely(work_done))
1485 goto out;
1486 return 0;
1487 }
1488
1489 do {
1490 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
1491 work_done +=
1492 mlx5e_decompress_cqes_start(rq, cqwq,
1493 budget - work_done);
1494 continue;
1495 }
1496
1497 mlx5_cqwq_pop(cqwq);
1498
1499 INDIRECT_CALL_2(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq,
1500 mlx5e_handle_rx_cqe, rq, cqe);
1501 } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));
1502
1503out:
1504 if (rq->xdp_prog)
1505 mlx5e_xdp_rx_poll_complete(rq);
1506
1507 mlx5_cqwq_update_db_record(cqwq);
1508
1509
1510 wmb();
1511
1512 return work_done;
1513}
1514
1515#ifdef CONFIG_MLX5_CORE_IPOIB
1516
1517#define MLX5_IB_GRH_SGID_OFFSET 8
1518#define MLX5_IB_GRH_DGID_OFFSET 24
1519#define MLX5_GID_SIZE 16
1520
1521static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
1522 struct mlx5_cqe64 *cqe,
1523 u32 cqe_bcnt,
1524 struct sk_buff *skb)
1525{
1526 struct hwtstamp_config *tstamp;
1527 struct mlx5e_rq_stats *stats;
1528 struct net_device *netdev;
1529 struct mlx5e_priv *priv;
1530 char *pseudo_header;
1531 u32 flags_rqpn;
1532 u32 qpn;
1533 u8 *dgid;
1534 u8 g;
1535
1536 qpn = be32_to_cpu(cqe->sop_drop_qpn) & 0xffffff;
1537 netdev = mlx5i_pkey_get_netdev(rq->netdev, qpn);
1538
1539
1540
1541
1542 if (unlikely(!netdev)) {
1543
1544 skb->dev = NULL;
1545 pr_warn_once("Unable to map QPN %u to dev - dropping skb\n", qpn);
1546 return;
1547 }
1548
1549 priv = mlx5i_epriv(netdev);
1550 tstamp = &priv->tstamp;
1551 stats = &priv->channel_stats[rq->ix].rq;
1552
1553 flags_rqpn = be32_to_cpu(cqe->flags_rqpn);
1554 g = (flags_rqpn >> 28) & 3;
1555 dgid = skb->data + MLX5_IB_GRH_DGID_OFFSET;
1556 if ((!g) || dgid[0] != 0xff)
1557 skb->pkt_type = PACKET_HOST;
1558 else if (memcmp(dgid, netdev->broadcast + 4, MLX5_GID_SIZE) == 0)
1559 skb->pkt_type = PACKET_BROADCAST;
1560 else
1561 skb->pkt_type = PACKET_MULTICAST;
1562
1563
1564
1565
1566 if (g && (qpn == (flags_rqpn & 0xffffff)) &&
1567 (memcmp(netdev->dev_addr + 4, skb->data + MLX5_IB_GRH_SGID_OFFSET,
1568 MLX5_GID_SIZE) == 0)) {
1569 skb->dev = NULL;
1570 return;
1571 }
1572
1573 skb_pull(skb, MLX5_IB_GRH_BYTES);
1574
1575 skb->protocol = *((__be16 *)(skb->data));
1576
1577 if (netdev->features & NETIF_F_RXCSUM) {
1578 skb->ip_summed = CHECKSUM_COMPLETE;
1579 skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
1580 stats->csum_complete++;
1581 } else {
1582 skb->ip_summed = CHECKSUM_NONE;
1583 stats->csum_none++;
1584 }
1585
1586 if (unlikely(mlx5e_rx_hw_stamp(tstamp)))
1587 skb_hwtstamps(skb)->hwtstamp =
1588 mlx5_timecounter_cyc2time(rq->clock, get_cqe_ts(cqe));
1589
1590 skb_record_rx_queue(skb, rq->ix);
1591
1592 if (likely(netdev->features & NETIF_F_RXHASH))
1593 mlx5e_skb_set_hash(cqe, skb);
1594
1595
1596 pseudo_header = skb_push(skb, MLX5_IPOIB_PSEUDO_LEN);
1597 memset(pseudo_header, 0, MLX5_IPOIB_PSEUDO_LEN);
1598 skb_reset_mac_header(skb);
1599 skb_pull(skb, MLX5_IPOIB_HARD_LEN);
1600
1601 skb->dev = netdev;
1602
1603 stats->packets++;
1604 stats->bytes += cqe_bcnt;
1605}
1606
1607void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1608{
1609 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1610 struct mlx5e_wqe_frag_info *wi;
1611 struct sk_buff *skb;
1612 u32 cqe_bcnt;
1613 u16 ci;
1614
1615 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1616 wi = get_frag(rq, ci);
1617 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1618
1619 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1620 rq->stats->wqe_err++;
1621 goto wq_free_wqe;
1622 }
1623
1624 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1625 mlx5e_skb_from_cqe_linear,
1626 mlx5e_skb_from_cqe_nonlinear,
1627 rq, cqe, wi, cqe_bcnt);
1628 if (!skb)
1629 goto wq_free_wqe;
1630
1631 mlx5i_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1632 if (unlikely(!skb->dev)) {
1633 dev_kfree_skb_any(skb);
1634 goto wq_free_wqe;
1635 }
1636 napi_gro_receive(rq->cq.napi, skb);
1637
1638wq_free_wqe:
1639 mlx5e_free_rx_wqe(rq, wi, true);
1640 mlx5_wq_cyc_pop(wq);
1641}
1642
1643#endif
1644
1645#ifdef CONFIG_MLX5_EN_IPSEC
1646
1647void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
1648{
1649 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
1650 struct mlx5e_wqe_frag_info *wi;
1651 struct sk_buff *skb;
1652 u32 cqe_bcnt;
1653 u16 ci;
1654
1655 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
1656 wi = get_frag(rq, ci);
1657 cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
1658
1659 if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
1660 rq->stats->wqe_err++;
1661 goto wq_free_wqe;
1662 }
1663
1664 skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
1665 mlx5e_skb_from_cqe_linear,
1666 mlx5e_skb_from_cqe_nonlinear,
1667 rq, cqe, wi, cqe_bcnt);
1668 if (unlikely(!skb))
1669 goto wq_free_wqe;
1670
1671 skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);
1672 if (unlikely(!skb))
1673 goto wq_free_wqe;
1674
1675 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1676 napi_gro_receive(rq->cq.napi, skb);
1677
1678wq_free_wqe:
1679 mlx5e_free_rx_wqe(rq, wi, true);
1680 mlx5_wq_cyc_pop(wq);
1681}
1682
1683#endif
1684