1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/tcp.h>
34#include <linux/if_vlan.h>
35#include <net/dsfield.h>
36#include "en.h"
37#include "ipoib/ipoib.h"
38#include "en_accel/en_accel.h"
39#include "lib/clock.h"
40
41#define MLX5E_SQ_NOPS_ROOM MLX5_SEND_WQE_MAX_WQEBBS
42
43#ifndef CONFIG_MLX5_EN_TLS
44#define MLX5E_SQ_STOP_ROOM (MLX5_SEND_WQE_MAX_WQEBBS +\
45 MLX5E_SQ_NOPS_ROOM)
46#else
47
48
49
50#define MLX5E_SQ_STOP_ROOM (2 * MLX5_SEND_WQE_MAX_WQEBBS +\
51 MLX5E_SQ_NOPS_ROOM)
52#endif
53
54static inline void mlx5e_tx_dma_unmap(struct device *pdev,
55 struct mlx5e_sq_dma *dma)
56{
57 switch (dma->type) {
58 case MLX5E_DMA_MAP_SINGLE:
59 dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
60 break;
61 case MLX5E_DMA_MAP_PAGE:
62 dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
63 break;
64 default:
65 WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
66 }
67}
68
69static inline struct mlx5e_sq_dma *mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
70{
71 return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
72}
73
74static inline void mlx5e_dma_push(struct mlx5e_txqsq *sq,
75 dma_addr_t addr,
76 u32 size,
77 enum mlx5e_dma_map_type map_type)
78{
79 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
80
81 dma->addr = addr;
82 dma->size = size;
83 dma->type = map_type;
84}
85
86static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)
87{
88 int i;
89
90 for (i = 0; i < num_dma; i++) {
91 struct mlx5e_sq_dma *last_pushed_dma =
92 mlx5e_dma_get(sq, --sq->dma_fifo_pc);
93
94 mlx5e_tx_dma_unmap(sq->pdev, last_pushed_dma);
95 }
96}
97
98#ifdef CONFIG_MLX5_CORE_EN_DCB
99static inline int mlx5e_get_dscp_up(struct mlx5e_priv *priv, struct sk_buff *skb)
100{
101 int dscp_cp = 0;
102
103 if (skb->protocol == htons(ETH_P_IP))
104 dscp_cp = ipv4_get_dsfield(ip_hdr(skb)) >> 2;
105 else if (skb->protocol == htons(ETH_P_IPV6))
106 dscp_cp = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2;
107
108 return priv->dcbx_dp.dscp2prio[dscp_cp];
109}
110#endif
111
112u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
113 void *accel_priv, select_queue_fallback_t fallback)
114{
115 struct mlx5e_priv *priv = netdev_priv(dev);
116 int txq_ix = fallback(dev, skb);
117 u16 num_channels;
118 int up = 0;
119
120 if (!netdev_get_num_tc(dev))
121 return txq_ix;
122
123#ifdef CONFIG_MLX5_CORE_EN_DCB
124 if (priv->dcbx_dp.trust_state == MLX5_QPTS_TRUST_DSCP)
125 up = mlx5e_get_dscp_up(priv, skb);
126 else
127#endif
128 if (skb_vlan_tag_present(skb))
129 up = skb->vlan_tci >> VLAN_PRIO_SHIFT;
130
131
132
133
134 num_channels = priv->channels.params.num_channels;
135 if (txq_ix >= num_channels)
136 txq_ix = priv->txq2sq[txq_ix]->ch_ix;
137
138 return priv->channel_tc2txq[txq_ix][up];
139}
140
141static inline int mlx5e_skb_l2_header_offset(struct sk_buff *skb)
142{
143#define MLX5E_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
144
145 return max(skb_network_offset(skb), MLX5E_MIN_INLINE);
146}
147
148static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb)
149{
150 struct flow_keys keys;
151
152 if (skb_transport_header_was_set(skb))
153 return skb_transport_offset(skb);
154 else if (skb_flow_dissect_flow_keys(skb, &keys, 0))
155 return keys.control.thoff;
156 else
157 return mlx5e_skb_l2_header_offset(skb);
158}
159
160static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode,
161 struct sk_buff *skb)
162{
163 u16 hlen;
164
165 switch (mode) {
166 case MLX5_INLINE_MODE_NONE:
167 return 0;
168 case MLX5_INLINE_MODE_TCP_UDP:
169 hlen = eth_get_headlen(skb->data, skb_headlen(skb));
170 if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb))
171 hlen += VLAN_HLEN;
172 break;
173 case MLX5_INLINE_MODE_IP:
174
175
176
177
178 if (skb_transport_offset(skb)) {
179 hlen = mlx5e_skb_l3_header_offset(skb);
180 break;
181 }
182
183 case MLX5_INLINE_MODE_L2:
184 default:
185 hlen = mlx5e_skb_l2_header_offset(skb);
186 }
187 return min_t(u16, hlen, skb_headlen(skb));
188}
189
190static inline void mlx5e_insert_vlan(void *start, struct sk_buff *skb, u16 ihs)
191{
192 struct vlan_ethhdr *vhdr = (struct vlan_ethhdr *)start;
193 int cpy1_sz = 2 * ETH_ALEN;
194 int cpy2_sz = ihs - cpy1_sz;
195
196 memcpy(vhdr, skb->data, cpy1_sz);
197 vhdr->h_vlan_proto = skb->vlan_proto;
198 vhdr->h_vlan_TCI = cpu_to_be16(skb_vlan_tag_get(skb));
199 memcpy(&vhdr->h_vlan_encapsulated_proto, skb->data + cpy1_sz, cpy2_sz);
200}
201
202static inline void
203mlx5e_txwqe_build_eseg_csum(struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg)
204{
205 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
206 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM;
207 if (skb->encapsulation) {
208 eseg->cs_flags |= MLX5_ETH_WQE_L3_INNER_CSUM |
209 MLX5_ETH_WQE_L4_INNER_CSUM;
210 sq->stats->csum_partial_inner++;
211 } else {
212 eseg->cs_flags |= MLX5_ETH_WQE_L4_CSUM;
213 sq->stats->csum_partial++;
214 }
215 } else
216 sq->stats->csum_none++;
217}
218
219static inline u16
220mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb)
221{
222 struct mlx5e_sq_stats *stats = sq->stats;
223 u16 ihs;
224
225 if (skb->encapsulation) {
226 ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
227 stats->tso_inner_packets++;
228 stats->tso_inner_bytes += skb->len - ihs;
229 } else {
230 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
231 ihs = skb_transport_offset(skb) + sizeof(struct udphdr);
232 else
233 ihs = skb_transport_offset(skb) + tcp_hdrlen(skb);
234 stats->tso_packets++;
235 stats->tso_bytes += skb->len - ihs;
236 }
237
238 return ihs;
239}
240
241static inline int
242mlx5e_txwqe_build_dsegs(struct mlx5e_txqsq *sq, struct sk_buff *skb,
243 unsigned char *skb_data, u16 headlen,
244 struct mlx5_wqe_data_seg *dseg)
245{
246 dma_addr_t dma_addr = 0;
247 u8 num_dma = 0;
248 int i;
249
250 if (headlen) {
251 dma_addr = dma_map_single(sq->pdev, skb_data, headlen,
252 DMA_TO_DEVICE);
253 if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
254 goto dma_unmap_wqe_err;
255
256 dseg->addr = cpu_to_be64(dma_addr);
257 dseg->lkey = sq->mkey_be;
258 dseg->byte_count = cpu_to_be32(headlen);
259
260 mlx5e_dma_push(sq, dma_addr, headlen, MLX5E_DMA_MAP_SINGLE);
261 num_dma++;
262 dseg++;
263 }
264
265 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
266 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
267 int fsz = skb_frag_size(frag);
268
269 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
270 DMA_TO_DEVICE);
271 if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
272 goto dma_unmap_wqe_err;
273
274 dseg->addr = cpu_to_be64(dma_addr);
275 dseg->lkey = sq->mkey_be;
276 dseg->byte_count = cpu_to_be32(fsz);
277
278 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE);
279 num_dma++;
280 dseg++;
281 }
282
283 return num_dma;
284
285dma_unmap_wqe_err:
286 mlx5e_dma_unmap_wqe_err(sq, num_dma);
287 return -ENOMEM;
288}
289
290static inline void mlx5e_fill_sq_frag_edge(struct mlx5e_txqsq *sq,
291 struct mlx5_wq_cyc *wq,
292 u16 pi, u16 nnops)
293{
294 struct mlx5e_tx_wqe_info *edge_wi, *wi = &sq->db.wqe_info[pi];
295
296 edge_wi = wi + nnops;
297
298
299 for (; wi < edge_wi; wi++) {
300 wi->skb = NULL;
301 wi->num_wqebbs = 1;
302 mlx5e_post_nop(wq, sq->sqn, &sq->pc);
303 }
304 sq->stats->nop += nnops;
305}
306
307static inline void
308mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb,
309 u8 opcode, u16 ds_cnt, u8 num_wqebbs, u32 num_bytes, u8 num_dma,
310 struct mlx5e_tx_wqe_info *wi, struct mlx5_wqe_ctrl_seg *cseg)
311{
312 struct mlx5_wq_cyc *wq = &sq->wq;
313
314 wi->num_bytes = num_bytes;
315 wi->num_dma = num_dma;
316 wi->num_wqebbs = num_wqebbs;
317 wi->skb = skb;
318
319 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
320 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
321
322 netdev_tx_sent_queue(sq->txq, num_bytes);
323
324 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
325 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
326
327 sq->pc += wi->num_wqebbs;
328 if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, MLX5E_SQ_STOP_ROOM))) {
329 netif_tx_stop_queue(sq->txq);
330 sq->stats->stopped++;
331 }
332
333 if (!skb->xmit_more || netif_xmit_stopped(sq->txq))
334 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, cseg);
335}
336
337#define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
338
339netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
340 struct mlx5e_tx_wqe *wqe, u16 pi)
341{
342 struct mlx5_wq_cyc *wq = &sq->wq;
343 struct mlx5_wqe_ctrl_seg *cseg;
344 struct mlx5_wqe_eth_seg *eseg;
345 struct mlx5_wqe_data_seg *dseg;
346 struct mlx5e_tx_wqe_info *wi;
347
348 struct mlx5e_sq_stats *stats = sq->stats;
349 u16 headlen, ihs, contig_wqebbs_room;
350 u16 ds_cnt, ds_cnt_inl = 0;
351 u8 num_wqebbs, opcode;
352 u32 num_bytes;
353 int num_dma;
354 __be16 mss;
355
356
357 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
358 if (skb_is_gso(skb)) {
359 opcode = MLX5_OPCODE_LSO;
360 mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
361 ihs = mlx5e_tx_get_gso_ihs(sq, skb);
362 num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
363 stats->packets += skb_shinfo(skb)->gso_segs;
364 } else {
365 opcode = MLX5_OPCODE_SEND;
366 mss = 0;
367 ihs = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
368 num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
369 stats->packets++;
370 }
371
372 stats->bytes += num_bytes;
373 stats->xmit_more += skb->xmit_more;
374
375 headlen = skb->len - ihs - skb->data_len;
376 ds_cnt += !!headlen;
377 ds_cnt += skb_shinfo(skb)->nr_frags;
378
379 if (ihs) {
380 ihs += !!skb_vlan_tag_present(skb) * VLAN_HLEN;
381
382 ds_cnt_inl = DIV_ROUND_UP(ihs - INL_HDR_START_SZ, MLX5_SEND_WQE_DS);
383 ds_cnt += ds_cnt_inl;
384 }
385
386 num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
387 contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
388 if (unlikely(contig_wqebbs_room < num_wqebbs)) {
389#ifdef CONFIG_MLX5_EN_IPSEC
390 struct mlx5_wqe_eth_seg cur_eth = wqe->eth;
391#endif
392 mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room);
393 mlx5e_sq_fetch_wqe(sq, &wqe, &pi);
394#ifdef CONFIG_MLX5_EN_IPSEC
395 wqe->eth = cur_eth;
396#endif
397 }
398
399
400 wi = &sq->db.wqe_info[pi];
401 cseg = &wqe->ctrl;
402 eseg = &wqe->eth;
403 dseg = wqe->data;
404
405 mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
406
407 eseg->mss = mss;
408
409 if (ihs) {
410 eseg->inline_hdr.sz = cpu_to_be16(ihs);
411 if (skb_vlan_tag_present(skb)) {
412 ihs -= VLAN_HLEN;
413 mlx5e_insert_vlan(eseg->inline_hdr.start, skb, ihs);
414 stats->added_vlan_packets++;
415 } else {
416 memcpy(eseg->inline_hdr.start, skb->data, ihs);
417 }
418 dseg += ds_cnt_inl;
419 } else if (skb_vlan_tag_present(skb)) {
420 eseg->insert.type = cpu_to_be16(MLX5_ETH_WQE_INSERT_VLAN);
421 if (skb->vlan_proto == cpu_to_be16(ETH_P_8021AD))
422 eseg->insert.type |= cpu_to_be16(MLX5_ETH_WQE_SVLAN);
423 eseg->insert.vlan_tci = cpu_to_be16(skb_vlan_tag_get(skb));
424 stats->added_vlan_packets++;
425 }
426
427 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + ihs, headlen, dseg);
428 if (unlikely(num_dma < 0))
429 goto err_drop;
430
431 mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes,
432 num_dma, wi, cseg);
433
434 return NETDEV_TX_OK;
435
436err_drop:
437 stats->dropped++;
438 dev_kfree_skb_any(skb);
439
440 return NETDEV_TX_OK;
441}
442
443netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
444{
445 struct mlx5e_priv *priv = netdev_priv(dev);
446 struct mlx5e_tx_wqe *wqe;
447 struct mlx5e_txqsq *sq;
448 u16 pi;
449
450 sq = priv->txq2sq[skb_get_queue_mapping(skb)];
451 mlx5e_sq_fetch_wqe(sq, &wqe, &pi);
452
453
454 skb = mlx5e_accel_handle_tx(skb, sq, dev, &wqe, &pi);
455 if (unlikely(!skb))
456 return NETDEV_TX_OK;
457
458 return mlx5e_sq_xmit(sq, skb, wqe, pi);
459}
460
461static void mlx5e_dump_error_cqe(struct mlx5e_txqsq *sq,
462 struct mlx5_err_cqe *err_cqe)
463{
464 u32 ci = mlx5_cqwq_get_ci(&sq->cq.wq);
465
466 netdev_err(sq->channel->netdev,
467 "Error cqe on cqn 0x%x, ci 0x%x, sqn 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
468 sq->cq.mcq.cqn, ci, sq->sqn, err_cqe->syndrome,
469 err_cqe->vendor_err_synd);
470 mlx5_dump_err_cqe(sq->cq.mdev, err_cqe);
471}
472
473bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
474{
475 struct mlx5e_sq_stats *stats;
476 struct mlx5e_txqsq *sq;
477 struct mlx5_cqe64 *cqe;
478 u32 dma_fifo_cc;
479 u32 nbytes;
480 u16 npkts;
481 u16 sqcc;
482 int i;
483
484 sq = container_of(cq, struct mlx5e_txqsq, cq);
485
486 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
487 return false;
488
489 cqe = mlx5_cqwq_get_cqe(&cq->wq);
490 if (!cqe)
491 return false;
492
493 stats = sq->stats;
494
495 npkts = 0;
496 nbytes = 0;
497
498
499
500
501 sqcc = sq->cc;
502
503
504 dma_fifo_cc = sq->dma_fifo_cc;
505
506 i = 0;
507 do {
508 u16 wqe_counter;
509 bool last_wqe;
510
511 mlx5_cqwq_pop(&cq->wq);
512
513 wqe_counter = be16_to_cpu(cqe->wqe_counter);
514
515 if (unlikely(cqe->op_own >> 4 == MLX5_CQE_REQ_ERR)) {
516 if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING,
517 &sq->state)) {
518 mlx5e_dump_error_cqe(sq,
519 (struct mlx5_err_cqe *)cqe);
520 queue_work(cq->channel->priv->wq,
521 &sq->recover.recover_work);
522 }
523 stats->cqe_err++;
524 }
525
526 do {
527 struct mlx5e_tx_wqe_info *wi;
528 struct sk_buff *skb;
529 u16 ci;
530 int j;
531
532 last_wqe = (sqcc == wqe_counter);
533
534 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
535 wi = &sq->db.wqe_info[ci];
536 skb = wi->skb;
537
538 if (unlikely(!skb)) {
539 sqcc++;
540 continue;
541 }
542
543 if (unlikely(skb_shinfo(skb)->tx_flags &
544 SKBTX_HW_TSTAMP)) {
545 struct skb_shared_hwtstamps hwts = {};
546
547 hwts.hwtstamp =
548 mlx5_timecounter_cyc2time(sq->clock,
549 get_cqe_ts(cqe));
550 skb_tstamp_tx(skb, &hwts);
551 }
552
553 for (j = 0; j < wi->num_dma; j++) {
554 struct mlx5e_sq_dma *dma =
555 mlx5e_dma_get(sq, dma_fifo_cc++);
556
557 mlx5e_tx_dma_unmap(sq->pdev, dma);
558 }
559
560 npkts++;
561 nbytes += wi->num_bytes;
562 sqcc += wi->num_wqebbs;
563 napi_consume_skb(skb, napi_budget);
564 } while (!last_wqe);
565
566 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
567
568 stats->cqes += i;
569
570 mlx5_cqwq_update_db_record(&cq->wq);
571
572
573 wmb();
574
575 sq->dma_fifo_cc = dma_fifo_cc;
576 sq->cc = sqcc;
577
578 netdev_tx_completed_queue(sq->txq, npkts, nbytes);
579
580 if (netif_tx_queue_stopped(sq->txq) &&
581 mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc,
582 MLX5E_SQ_STOP_ROOM) &&
583 !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) {
584 netif_tx_wake_queue(sq->txq);
585 stats->wake++;
586 }
587
588 return (i == MLX5E_TX_CQ_POLL_BUDGET);
589}
590
591void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq)
592{
593 struct mlx5e_tx_wqe_info *wi;
594 struct sk_buff *skb;
595 u16 ci;
596 int i;
597
598 while (sq->cc != sq->pc) {
599 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
600 wi = &sq->db.wqe_info[ci];
601 skb = wi->skb;
602
603 if (!skb) {
604 sq->cc++;
605 continue;
606 }
607
608 for (i = 0; i < wi->num_dma; i++) {
609 struct mlx5e_sq_dma *dma =
610 mlx5e_dma_get(sq, sq->dma_fifo_cc++);
611
612 mlx5e_tx_dma_unmap(sq->pdev, dma);
613 }
614
615 dev_kfree_skb_any(skb);
616 sq->cc += wi->num_wqebbs;
617 }
618}
619
620#ifdef CONFIG_MLX5_CORE_IPOIB
621static inline void
622mlx5i_txwqe_build_datagram(struct mlx5_av *av, u32 dqpn, u32 dqkey,
623 struct mlx5_wqe_datagram_seg *dseg)
624{
625 memcpy(&dseg->av, av, sizeof(struct mlx5_av));
626 dseg->av.dqp_dct = cpu_to_be32(dqpn | MLX5_EXTENDED_UD_AV);
627 dseg->av.key.qkey.qkey = cpu_to_be32(dqkey);
628}
629
630netdev_tx_t mlx5i_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
631 struct mlx5_av *av, u32 dqpn, u32 dqkey)
632{
633 struct mlx5_wq_cyc *wq = &sq->wq;
634 struct mlx5i_tx_wqe *wqe;
635
636 struct mlx5_wqe_datagram_seg *datagram;
637 struct mlx5_wqe_ctrl_seg *cseg;
638 struct mlx5_wqe_eth_seg *eseg;
639 struct mlx5_wqe_data_seg *dseg;
640 struct mlx5e_tx_wqe_info *wi;
641
642 struct mlx5e_sq_stats *stats = sq->stats;
643 u16 headlen, ihs, pi, contig_wqebbs_room;
644 u16 ds_cnt, ds_cnt_inl = 0;
645 u8 num_wqebbs, opcode;
646 u32 num_bytes;
647 int num_dma;
648 __be16 mss;
649
650
651 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
652 if (skb_is_gso(skb)) {
653 opcode = MLX5_OPCODE_LSO;
654 mss = cpu_to_be16(skb_shinfo(skb)->gso_size);
655 ihs = mlx5e_tx_get_gso_ihs(sq, skb);
656 num_bytes = skb->len + (skb_shinfo(skb)->gso_segs - 1) * ihs;
657 stats->packets += skb_shinfo(skb)->gso_segs;
658 } else {
659 opcode = MLX5_OPCODE_SEND;
660 mss = 0;
661 ihs = mlx5e_calc_min_inline(sq->min_inline_mode, skb);
662 num_bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
663 stats->packets++;
664 }
665
666 stats->bytes += num_bytes;
667 stats->xmit_more += skb->xmit_more;
668
669 headlen = skb->len - ihs - skb->data_len;
670 ds_cnt += !!headlen;
671 ds_cnt += skb_shinfo(skb)->nr_frags;
672
673 if (ihs) {
674 ds_cnt_inl = DIV_ROUND_UP(ihs - INL_HDR_START_SZ, MLX5_SEND_WQE_DS);
675 ds_cnt += ds_cnt_inl;
676 }
677
678 num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
679 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
680 contig_wqebbs_room = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
681 if (unlikely(contig_wqebbs_room < num_wqebbs)) {
682 mlx5e_fill_sq_frag_edge(sq, wq, pi, contig_wqebbs_room);
683 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
684 }
685
686 mlx5i_sq_fetch_wqe(sq, &wqe, pi);
687
688
689 wi = &sq->db.wqe_info[pi];
690 cseg = &wqe->ctrl;
691 datagram = &wqe->datagram;
692 eseg = &wqe->eth;
693 dseg = wqe->data;
694
695 mlx5i_txwqe_build_datagram(av, dqpn, dqkey, datagram);
696
697 mlx5e_txwqe_build_eseg_csum(sq, skb, eseg);
698
699 eseg->mss = mss;
700
701 if (ihs) {
702 memcpy(eseg->inline_hdr.start, skb->data, ihs);
703 eseg->inline_hdr.sz = cpu_to_be16(ihs);
704 dseg += ds_cnt_inl;
705 }
706
707 num_dma = mlx5e_txwqe_build_dsegs(sq, skb, skb->data + ihs, headlen, dseg);
708 if (unlikely(num_dma < 0))
709 goto err_drop;
710
711 mlx5e_txwqe_complete(sq, skb, opcode, ds_cnt, num_wqebbs, num_bytes,
712 num_dma, wi, cseg);
713
714 return NETDEV_TX_OK;
715
716err_drop:
717 stats->dropped++;
718 dev_kfree_skb_any(skb);
719
720 return NETDEV_TX_OK;
721}
722#endif
723
