1
2
3
4
5
6#include <sys/queue.h>
7
8#include <stdio.h>
9#include <stdlib.h>
10#include <string.h>
11#include <errno.h>
12#include <stdint.h>
13#include <stdarg.h>
14#include <unistd.h>
15#include <inttypes.h>
16
17#include <rte_byteorder.h>
18#include <rte_common.h>
19#include <rte_cycles.h>
20#include <rte_log.h>
21#include <rte_debug.h>
22#include <rte_ethdev.h>
23#include <ethdev_driver.h>
24#include <rte_security_driver.h>
25#include <rte_memzone.h>
26#include <rte_atomic.h>
27#include <rte_mempool.h>
28#include <rte_malloc.h>
29#include <rte_mbuf.h>
30#include <rte_ether.h>
31#include <rte_prefetch.h>
32#include <rte_udp.h>
33#include <rte_tcp.h>
34#include <rte_sctp.h>
35#include <rte_string_fns.h>
36#include <rte_errno.h>
37#include <rte_ip.h>
38#include <rte_net.h>
39
40#include "txgbe_logs.h"
41#include "base/txgbe.h"
42#include "txgbe_ethdev.h"
43#include "txgbe_rxtx.h"
44
45#ifdef RTE_LIBRTE_IEEE1588
46#define TXGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST
47#else
48#define TXGBE_TX_IEEE1588_TMST 0
49#endif
50
51
52static const u64 TXGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM |
53 RTE_MBUF_F_TX_OUTER_IPV6 |
54 RTE_MBUF_F_TX_OUTER_IPV4 |
55 RTE_MBUF_F_TX_IPV6 |
56 RTE_MBUF_F_TX_IPV4 |
57 RTE_MBUF_F_TX_VLAN |
58 RTE_MBUF_F_TX_L4_MASK |
59 RTE_MBUF_F_TX_TCP_SEG |
60 RTE_MBUF_F_TX_TUNNEL_MASK |
61 RTE_MBUF_F_TX_OUTER_IP_CKSUM |
62 RTE_MBUF_F_TX_OUTER_UDP_CKSUM |
63#ifdef RTE_LIB_SECURITY
64 RTE_MBUF_F_TX_SEC_OFFLOAD |
65#endif
66 TXGBE_TX_IEEE1588_TMST);
67
68#define TXGBE_TX_OFFLOAD_NOTSUP_MASK \
69 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK)
70
71
72
73
74#define rte_txgbe_prefetch(p) rte_prefetch0(p)
75
76static int
77txgbe_is_vf(struct rte_eth_dev *dev)
78{
79 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
80
81 switch (hw->mac.type) {
82 case txgbe_mac_raptor_vf:
83 return 1;
84 default:
85 return 0;
86 }
87}
88
89
90
91
92
93
94
95
96
97
98
99static __rte_always_inline int
100txgbe_tx_free_bufs(struct txgbe_tx_queue *txq)
101{
102 struct txgbe_tx_entry *txep;
103 uint32_t status;
104 int i, nb_free = 0;
105 struct rte_mbuf *m, *free[RTE_TXGBE_TX_MAX_FREE_BUF_SZ];
106
107
108 status = txq->tx_ring[txq->tx_next_dd].dw3;
109 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
110 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
111 txgbe_set32_masked(txq->tdc_reg_addr,
112 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
113 return 0;
114 }
115
116
117
118
119
120 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
121 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
122
123 m = rte_pktmbuf_prefree_seg(txep->mbuf);
124 txep->mbuf = NULL;
125
126 if (unlikely(m == NULL))
127 continue;
128
129 if (nb_free >= RTE_TXGBE_TX_MAX_FREE_BUF_SZ ||
130 (nb_free > 0 && m->pool != free[0]->pool)) {
131 rte_mempool_put_bulk(free[0]->pool,
132 (void **)free, nb_free);
133 nb_free = 0;
134 }
135
136 free[nb_free++] = m;
137 }
138
139 if (nb_free > 0)
140 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
141
142
143 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
144 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
145 if (txq->tx_next_dd >= txq->nb_tx_desc)
146 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
147
148 return txq->tx_free_thresh;
149}
150
151
152static inline void
153tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
154{
155 uint64_t buf_dma_addr;
156 uint32_t pkt_len;
157 int i;
158
159 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
160 buf_dma_addr = rte_mbuf_data_iova(*pkts);
161 pkt_len = (*pkts)->data_len;
162
163
164 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
165 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
166 TXGBE_TXD_DATLEN(pkt_len));
167 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
168
169 rte_prefetch0(&(*pkts)->pool);
170 }
171}
172
173
174static inline void
175tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
176{
177 uint64_t buf_dma_addr;
178 uint32_t pkt_len;
179
180 buf_dma_addr = rte_mbuf_data_iova(*pkts);
181 pkt_len = (*pkts)->data_len;
182
183
184 txdp->qw0 = cpu_to_le64(buf_dma_addr);
185 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
186 TXGBE_TXD_DATLEN(pkt_len));
187 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
188
189 rte_prefetch0(&(*pkts)->pool);
190}
191
192
193
194
195
196static inline void
197txgbe_tx_fill_hw_ring(struct txgbe_tx_queue *txq, struct rte_mbuf **pkts,
198 uint16_t nb_pkts)
199{
200 volatile struct txgbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
201 struct txgbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
202 const int N_PER_LOOP = 4;
203 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
204 int mainpart, leftover;
205 int i, j;
206
207
208
209
210
211 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
212 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
213 for (i = 0; i < mainpart; i += N_PER_LOOP) {
214
215 for (j = 0; j < N_PER_LOOP; ++j)
216 (txep + i + j)->mbuf = *(pkts + i + j);
217 tx4(txdp + i, pkts + i);
218 }
219
220 if (unlikely(leftover > 0)) {
221 for (i = 0; i < leftover; ++i) {
222 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
223 tx1(txdp + mainpart + i, pkts + mainpart + i);
224 }
225 }
226}
227
228static inline uint16_t
229tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
230 uint16_t nb_pkts)
231{
232 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
233 uint16_t n = 0;
234
235
236
237
238
239
240 if (txq->nb_tx_free < txq->tx_free_thresh)
241 txgbe_tx_free_bufs(txq);
242
243
244 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
245 if (unlikely(nb_pkts == 0))
246 return 0;
247
248
249 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
250
251
252
253
254
255
256
257
258
259
260
261
262
263 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
264 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
265 txgbe_tx_fill_hw_ring(txq, tx_pkts, n);
266 txq->tx_tail = 0;
267 }
268
269
270 txgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
271 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
272
273
274
275
276
277 if (txq->tx_tail >= txq->nb_tx_desc)
278 txq->tx_tail = 0;
279
280 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
281 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
282 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
283
284
285 rte_wmb();
286 txgbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
287
288 return nb_pkts;
289}
290
291uint16_t
292txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
293 uint16_t nb_pkts)
294{
295 uint16_t nb_tx;
296
297
298 if (likely(nb_pkts <= RTE_PMD_TXGBE_TX_MAX_BURST))
299 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
300
301
302 nb_tx = 0;
303 while (nb_pkts) {
304 uint16_t ret, n;
305
306 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_TX_MAX_BURST);
307 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
308 nb_tx = (uint16_t)(nb_tx + ret);
309 nb_pkts = (uint16_t)(nb_pkts - ret);
310 if (ret < n)
311 break;
312 }
313
314 return nb_tx;
315}
316
317static inline void
318txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq,
319 volatile struct txgbe_tx_ctx_desc *ctx_txd,
320 uint64_t ol_flags, union txgbe_tx_offload tx_offload,
321 __rte_unused uint64_t *mdata)
322{
323 union txgbe_tx_offload tx_offload_mask;
324 uint32_t type_tucmd_mlhl;
325 uint32_t mss_l4len_idx;
326 uint32_t ctx_idx;
327 uint32_t vlan_macip_lens;
328 uint32_t tunnel_seed;
329
330 ctx_idx = txq->ctx_curr;
331 tx_offload_mask.data[0] = 0;
332 tx_offload_mask.data[1] = 0;
333
334
335 mss_l4len_idx = TXGBE_TXD_IDX(ctx_idx);
336 type_tucmd_mlhl = TXGBE_TXD_CTXT;
337
338 tx_offload_mask.ptid |= ~0;
339 type_tucmd_mlhl |= TXGBE_TXD_PTID(tx_offload.ptid);
340
341
342 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
343 tx_offload_mask.l2_len |= ~0;
344 tx_offload_mask.l3_len |= ~0;
345 tx_offload_mask.l4_len |= ~0;
346 tx_offload_mask.tso_segsz |= ~0;
347 mss_l4len_idx |= TXGBE_TXD_MSS(tx_offload.tso_segsz);
348 mss_l4len_idx |= TXGBE_TXD_L4LEN(tx_offload.l4_len);
349 } else {
350 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
351 tx_offload_mask.l2_len |= ~0;
352 tx_offload_mask.l3_len |= ~0;
353 }
354
355 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
356 case RTE_MBUF_F_TX_UDP_CKSUM:
357 mss_l4len_idx |=
358 TXGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr));
359 tx_offload_mask.l2_len |= ~0;
360 tx_offload_mask.l3_len |= ~0;
361 break;
362 case RTE_MBUF_F_TX_TCP_CKSUM:
363 mss_l4len_idx |=
364 TXGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr));
365 tx_offload_mask.l2_len |= ~0;
366 tx_offload_mask.l3_len |= ~0;
367 break;
368 case RTE_MBUF_F_TX_SCTP_CKSUM:
369 mss_l4len_idx |=
370 TXGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr));
371 tx_offload_mask.l2_len |= ~0;
372 tx_offload_mask.l3_len |= ~0;
373 break;
374 default:
375 break;
376 }
377 }
378
379 vlan_macip_lens = TXGBE_TXD_IPLEN(tx_offload.l3_len >> 1);
380
381 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
382 tx_offload_mask.outer_tun_len |= ~0;
383 tx_offload_mask.outer_l2_len |= ~0;
384 tx_offload_mask.outer_l3_len |= ~0;
385 tx_offload_mask.l2_len |= ~0;
386 tunnel_seed = TXGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1);
387 tunnel_seed |= TXGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2);
388
389 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
390 case RTE_MBUF_F_TX_TUNNEL_IPIP:
391
392 break;
393 case RTE_MBUF_F_TX_TUNNEL_VXLAN:
394 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
395 case RTE_MBUF_F_TX_TUNNEL_GENEVE:
396 tunnel_seed |= TXGBE_TXD_ETYPE_UDP;
397 break;
398 case RTE_MBUF_F_TX_TUNNEL_GRE:
399 tunnel_seed |= TXGBE_TXD_ETYPE_GRE;
400 break;
401 default:
402 PMD_TX_LOG(ERR, "Tunnel type not supported");
403 return;
404 }
405 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.outer_l2_len);
406 } else {
407 tunnel_seed = 0;
408 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.l2_len);
409 }
410
411 if (ol_flags & RTE_MBUF_F_TX_VLAN) {
412 tx_offload_mask.vlan_tci |= ~0;
413 vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci);
414 }
415
416#ifdef RTE_LIB_SECURITY
417 if (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) {
418 union txgbe_crypto_tx_desc_md *md =
419 (union txgbe_crypto_tx_desc_md *)mdata;
420 tunnel_seed |= TXGBE_TXD_IPSEC_SAIDX(md->sa_idx);
421 type_tucmd_mlhl |= md->enc ?
422 (TXGBE_TXD_IPSEC_ESP | TXGBE_TXD_IPSEC_ESPENC) : 0;
423 type_tucmd_mlhl |= TXGBE_TXD_IPSEC_ESPLEN(md->pad_len);
424 tx_offload_mask.sa_idx |= ~0;
425 tx_offload_mask.sec_pad_len |= ~0;
426 }
427#endif
428
429 txq->ctx_cache[ctx_idx].flags = ol_flags;
430 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
431 tx_offload_mask.data[0] & tx_offload.data[0];
432 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
433 tx_offload_mask.data[1] & tx_offload.data[1];
434 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
435
436 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens);
437 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed);
438 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl);
439 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx);
440}
441
442
443
444
445
446static inline uint32_t
447what_ctx_update(struct txgbe_tx_queue *txq, uint64_t flags,
448 union txgbe_tx_offload tx_offload)
449{
450
451 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
452 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
453 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
454 & tx_offload.data[0])) &&
455 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
456 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
457 & tx_offload.data[1]))))
458 return txq->ctx_curr;
459
460
461 txq->ctx_curr ^= 1;
462 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
463 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
464 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
465 & tx_offload.data[0])) &&
466 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
467 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
468 & tx_offload.data[1]))))
469 return txq->ctx_curr;
470
471
472 return TXGBE_CTX_NUM;
473}
474
475static inline uint32_t
476tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
477{
478 uint32_t tmp = 0;
479
480 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) {
481 tmp |= TXGBE_TXD_CC;
482 tmp |= TXGBE_TXD_L4CS;
483 }
484 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
485 tmp |= TXGBE_TXD_CC;
486 tmp |= TXGBE_TXD_IPCS;
487 }
488 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) {
489 tmp |= TXGBE_TXD_CC;
490 tmp |= TXGBE_TXD_EIPCS;
491 }
492 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
493 tmp |= TXGBE_TXD_CC;
494
495 if (ol_flags & RTE_MBUF_F_TX_IPV4)
496 tmp |= TXGBE_TXD_IPCS;
497 tmp |= TXGBE_TXD_L4CS;
498 }
499 if (ol_flags & RTE_MBUF_F_TX_VLAN)
500 tmp |= TXGBE_TXD_CC;
501
502 return tmp;
503}
504
505static inline uint32_t
506tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
507{
508 uint32_t cmdtype = 0;
509
510 if (ol_flags & RTE_MBUF_F_TX_VLAN)
511 cmdtype |= TXGBE_TXD_VLE;
512 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
513 cmdtype |= TXGBE_TXD_TSE;
514 if (ol_flags & RTE_MBUF_F_TX_MACSEC)
515 cmdtype |= TXGBE_TXD_LINKSEC;
516 return cmdtype;
517}
518
519static inline uint8_t
520tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype)
521{
522 bool tun;
523
524 if (ptype)
525 return txgbe_encode_ptype(ptype);
526
527
528 tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK);
529
530
531 ptype = RTE_PTYPE_L2_ETHER;
532 if (oflags & RTE_MBUF_F_TX_VLAN)
533 ptype |= RTE_PTYPE_L2_ETHER_VLAN;
534
535
536 if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM))
537 ptype |= RTE_PTYPE_L3_IPV4;
538 else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6))
539 ptype |= RTE_PTYPE_L3_IPV6;
540
541 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM))
542 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4);
543 else if (oflags & (RTE_MBUF_F_TX_IPV6))
544 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6);
545
546
547 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) {
548 case RTE_MBUF_F_TX_TCP_CKSUM:
549 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
550 break;
551 case RTE_MBUF_F_TX_UDP_CKSUM:
552 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP);
553 break;
554 case RTE_MBUF_F_TX_SCTP_CKSUM:
555 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP);
556 break;
557 }
558
559 if (oflags & RTE_MBUF_F_TX_TCP_SEG)
560 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
561
562
563 switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) {
564 case RTE_MBUF_F_TX_TUNNEL_VXLAN:
565 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
566 ptype |= RTE_PTYPE_L2_ETHER |
567 RTE_PTYPE_L3_IPV4 |
568 RTE_PTYPE_TUNNEL_GRENAT;
569 break;
570 case RTE_MBUF_F_TX_TUNNEL_GRE:
571 ptype |= RTE_PTYPE_L2_ETHER |
572 RTE_PTYPE_L3_IPV4 |
573 RTE_PTYPE_TUNNEL_GRE;
574 ptype |= RTE_PTYPE_INNER_L2_ETHER;
575 break;
576 case RTE_MBUF_F_TX_TUNNEL_GENEVE:
577 ptype |= RTE_PTYPE_L2_ETHER |
578 RTE_PTYPE_L3_IPV4 |
579 RTE_PTYPE_TUNNEL_GENEVE;
580 ptype |= RTE_PTYPE_INNER_L2_ETHER;
581 break;
582 case RTE_MBUF_F_TX_TUNNEL_IPIP:
583 case RTE_MBUF_F_TX_TUNNEL_IP:
584 ptype |= RTE_PTYPE_L2_ETHER |
585 RTE_PTYPE_L3_IPV4 |
586 RTE_PTYPE_TUNNEL_IP;
587 break;
588 }
589
590 return txgbe_encode_ptype(ptype);
591}
592
593#ifndef DEFAULT_TX_FREE_THRESH
594#define DEFAULT_TX_FREE_THRESH 32
595#endif
596
597
598static inline int
599txgbe_xmit_cleanup(struct txgbe_tx_queue *txq)
600{
601 struct txgbe_tx_entry *sw_ring = txq->sw_ring;
602 volatile struct txgbe_tx_desc *txr = txq->tx_ring;
603 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
604 uint16_t nb_tx_desc = txq->nb_tx_desc;
605 uint16_t desc_to_clean_to;
606 uint16_t nb_tx_to_clean;
607 uint32_t status;
608
609
610 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh);
611 if (desc_to_clean_to >= nb_tx_desc)
612 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
613
614
615 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
616 status = txr[desc_to_clean_to].dw3;
617 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
618 PMD_TX_FREE_LOG(DEBUG,
619 "TX descriptor %4u is not done"
620 "(port=%d queue=%d)",
621 desc_to_clean_to,
622 txq->port_id, txq->queue_id);
623 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
624 txgbe_set32_masked(txq->tdc_reg_addr,
625 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
626
627 return -(1);
628 }
629
630
631 if (last_desc_cleaned > desc_to_clean_to)
632 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
633 desc_to_clean_to);
634 else
635 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
636 last_desc_cleaned);
637
638 PMD_TX_FREE_LOG(DEBUG,
639 "Cleaning %4u TX descriptors: %4u to %4u "
640 "(port=%d queue=%d)",
641 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
642 txq->port_id, txq->queue_id);
643
644
645
646
647
648
649
650 txr[desc_to_clean_to].dw3 = 0;
651
652
653 txq->last_desc_cleaned = desc_to_clean_to;
654 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
655
656
657 return 0;
658}
659
660static inline uint8_t
661txgbe_get_tun_len(struct rte_mbuf *mbuf)
662{
663 struct txgbe_genevehdr genevehdr;
664 const struct txgbe_genevehdr *gh;
665 uint8_t tun_len;
666
667 switch (mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
668 case RTE_MBUF_F_TX_TUNNEL_IPIP:
669 tun_len = 0;
670 break;
671 case RTE_MBUF_F_TX_TUNNEL_VXLAN:
672 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
673 tun_len = sizeof(struct txgbe_udphdr)
674 + sizeof(struct txgbe_vxlanhdr);
675 break;
676 case RTE_MBUF_F_TX_TUNNEL_GRE:
677 tun_len = sizeof(struct txgbe_nvgrehdr);
678 break;
679 case RTE_MBUF_F_TX_TUNNEL_GENEVE:
680 gh = rte_pktmbuf_read(mbuf,
681 mbuf->outer_l2_len + mbuf->outer_l3_len,
682 sizeof(genevehdr), &genevehdr);
683 tun_len = sizeof(struct txgbe_udphdr)
684 + sizeof(struct txgbe_genevehdr)
685 + (gh->opt_len << 2);
686 break;
687 default:
688 tun_len = 0;
689 }
690
691 return tun_len;
692}
693
694static inline uint8_t
695txgbe_parse_tun_ptid(struct rte_mbuf *tx_pkt)
696{
697 uint64_t l2_none, l2_mac, l2_mac_vlan;
698 uint8_t ptid = 0;
699
700 if ((tx_pkt->ol_flags & (RTE_MBUF_F_TX_TUNNEL_VXLAN |
701 RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE)) == 0)
702 return ptid;
703
704 l2_none = sizeof(struct txgbe_udphdr) + sizeof(struct txgbe_vxlanhdr);
705 l2_mac = l2_none + sizeof(struct rte_ether_hdr);
706 l2_mac_vlan = l2_mac + sizeof(struct rte_vlan_hdr);
707
708 if (tx_pkt->l2_len == l2_none)
709 ptid = TXGBE_PTID_TUN_EIG;
710 else if (tx_pkt->l2_len == l2_mac)
711 ptid = TXGBE_PTID_TUN_EIGM;
712 else if (tx_pkt->l2_len == l2_mac_vlan)
713 ptid = TXGBE_PTID_TUN_EIGMV;
714
715 return ptid;
716}
717
718uint16_t
719txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
720 uint16_t nb_pkts)
721{
722 struct txgbe_tx_queue *txq;
723 struct txgbe_tx_entry *sw_ring;
724 struct txgbe_tx_entry *txe, *txn;
725 volatile struct txgbe_tx_desc *txr;
726 volatile struct txgbe_tx_desc *txd;
727 struct rte_mbuf *tx_pkt;
728 struct rte_mbuf *m_seg;
729 uint64_t buf_dma_addr;
730 uint32_t olinfo_status;
731 uint32_t cmd_type_len;
732 uint32_t pkt_len;
733 uint16_t slen;
734 uint64_t ol_flags;
735 uint16_t tx_id;
736 uint16_t tx_last;
737 uint16_t nb_tx;
738 uint16_t nb_used;
739 uint64_t tx_ol_req;
740 uint32_t ctx = 0;
741 uint32_t new_ctx;
742 union txgbe_tx_offload tx_offload;
743#ifdef RTE_LIB_SECURITY
744 uint8_t use_ipsec;
745#endif
746
747 tx_offload.data[0] = 0;
748 tx_offload.data[1] = 0;
749 txq = tx_queue;
750 sw_ring = txq->sw_ring;
751 txr = txq->tx_ring;
752 tx_id = txq->tx_tail;
753 txe = &sw_ring[tx_id];
754
755
756 if (txq->nb_tx_free < txq->tx_free_thresh)
757 txgbe_xmit_cleanup(txq);
758
759 rte_prefetch0(&txe->mbuf->pool);
760
761
762 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
763 new_ctx = 0;
764 tx_pkt = *tx_pkts++;
765 pkt_len = tx_pkt->pkt_len;
766
767
768
769
770
771 ol_flags = tx_pkt->ol_flags;
772#ifdef RTE_LIB_SECURITY
773 use_ipsec = txq->using_ipsec && (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD);
774#endif
775
776
777 tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK;
778 if (tx_ol_req) {
779 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req,
780 tx_pkt->packet_type);
781 if (tx_offload.ptid & TXGBE_PTID_PKT_TUN)
782 tx_offload.ptid |= txgbe_parse_tun_ptid(tx_pkt);
783 tx_offload.l2_len = tx_pkt->l2_len;
784 tx_offload.l3_len = tx_pkt->l3_len;
785 tx_offload.l4_len = tx_pkt->l4_len;
786 tx_offload.vlan_tci = tx_pkt->vlan_tci;
787 tx_offload.tso_segsz = tx_pkt->tso_segsz;
788 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
789 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
790 tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt);
791
792#ifdef RTE_LIB_SECURITY
793 if (use_ipsec) {
794 union txgbe_crypto_tx_desc_md *ipsec_mdata =
795 (union txgbe_crypto_tx_desc_md *)
796 rte_security_dynfield(tx_pkt);
797 tx_offload.sa_idx = ipsec_mdata->sa_idx;
798 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
799 }
800#endif
801
802
803 ctx = what_ctx_update(txq, tx_ol_req, tx_offload);
804
805 new_ctx = (ctx == TXGBE_CTX_NUM);
806 ctx = txq->ctx_curr;
807 }
808
809
810
811
812
813
814 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
815
816
817
818
819
820
821
822
823
824 tx_last = (uint16_t)(tx_id + nb_used - 1);
825
826
827 if (tx_last >= txq->nb_tx_desc)
828 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
829
830 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
831 " tx_first=%u tx_last=%u",
832 (uint16_t)txq->port_id,
833 (uint16_t)txq->queue_id,
834 (uint32_t)pkt_len,
835 (uint16_t)tx_id,
836 (uint16_t)tx_last);
837
838
839
840
841
842
843 if (nb_used > txq->nb_tx_free) {
844 PMD_TX_FREE_LOG(DEBUG,
845 "Not enough free TX descriptors "
846 "nb_used=%4u nb_free=%4u "
847 "(port=%d queue=%d)",
848 nb_used, txq->nb_tx_free,
849 txq->port_id, txq->queue_id);
850
851 if (txgbe_xmit_cleanup(txq) != 0) {
852
853 if (nb_tx == 0)
854 return 0;
855 goto end_of_tx;
856 }
857
858
859 if (unlikely(nb_used > txq->tx_free_thresh)) {
860 PMD_TX_FREE_LOG(DEBUG,
861 "The number of descriptors needed to "
862 "transmit the packet exceeds the "
863 "RS bit threshold. This will impact "
864 "performance."
865 "nb_used=%4u nb_free=%4u "
866 "tx_free_thresh=%4u. "
867 "(port=%d queue=%d)",
868 nb_used, txq->nb_tx_free,
869 txq->tx_free_thresh,
870 txq->port_id, txq->queue_id);
871
872
873
874
875
876 while (nb_used > txq->nb_tx_free) {
877 if (txgbe_xmit_cleanup(txq) != 0) {
878
879
880
881
882 if (nb_tx == 0)
883 return 0;
884 goto end_of_tx;
885 }
886 }
887 }
888 }
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916 cmd_type_len = TXGBE_TXD_FCS;
917
918#ifdef RTE_LIBRTE_IEEE1588
919 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST)
920 cmd_type_len |= TXGBE_TXD_1588;
921#endif
922
923 olinfo_status = 0;
924 if (tx_ol_req) {
925 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
926
927
928
929 pkt_len -= (tx_offload.l2_len +
930 tx_offload.l3_len + tx_offload.l4_len);
931 pkt_len -=
932 (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
933 ? tx_offload.outer_l2_len +
934 tx_offload.outer_l3_len : 0;
935 }
936
937
938
939
940 if (new_ctx) {
941 volatile struct txgbe_tx_ctx_desc *ctx_txd;
942
943 ctx_txd = (volatile struct txgbe_tx_ctx_desc *)
944 &txr[tx_id];
945
946 txn = &sw_ring[txe->next_id];
947 rte_prefetch0(&txn->mbuf->pool);
948
949 if (txe->mbuf != NULL) {
950 rte_pktmbuf_free_seg(txe->mbuf);
951 txe->mbuf = NULL;
952 }
953
954 txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
955 tx_offload,
956 rte_security_dynfield(tx_pkt));
957
958 txe->last_id = tx_last;
959 tx_id = txe->next_id;
960 txe = txn;
961 }
962
963
964
965
966
967
968 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
969 olinfo_status |=
970 tx_desc_cksum_flags_to_olinfo(ol_flags);
971 olinfo_status |= TXGBE_TXD_IDX(ctx);
972 }
973
974 olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len);
975#ifdef RTE_LIB_SECURITY
976 if (use_ipsec)
977 olinfo_status |= TXGBE_TXD_IPSEC;
978#endif
979
980 m_seg = tx_pkt;
981 do {
982 txd = &txr[tx_id];
983 txn = &sw_ring[txe->next_id];
984 rte_prefetch0(&txn->mbuf->pool);
985
986 if (txe->mbuf != NULL)
987 rte_pktmbuf_free_seg(txe->mbuf);
988 txe->mbuf = m_seg;
989
990
991
992
993 slen = m_seg->data_len;
994 buf_dma_addr = rte_mbuf_data_iova(m_seg);
995 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr);
996 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen);
997 txd->dw3 = rte_cpu_to_le_32(olinfo_status);
998 txe->last_id = tx_last;
999 tx_id = txe->next_id;
1000 txe = txn;
1001 m_seg = m_seg->next;
1002 } while (m_seg != NULL);
1003
1004
1005
1006
1007 cmd_type_len |= TXGBE_TXD_EOP;
1008 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
1009
1010 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len);
1011 }
1012
1013end_of_tx:
1014
1015 rte_wmb();
1016
1017
1018
1019
1020 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
1021 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
1022 (uint16_t)tx_id, (uint16_t)nb_tx);
1023 txgbe_set32_relaxed(txq->tdt_reg_addr, tx_id);
1024 txq->tx_tail = tx_id;
1025
1026 return nb_tx;
1027}
1028
1029
1030
1031
1032
1033
1034uint16_t
1035txgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1036{
1037 int i, ret;
1038 uint64_t ol_flags;
1039 struct rte_mbuf *m;
1040 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
1041
1042 for (i = 0; i < nb_pkts; i++) {
1043 m = tx_pkts[i];
1044 ol_flags = m->ol_flags;
1045
1046
1047
1048
1049
1050
1051
1052
1053 if (m->nb_segs > TXGBE_TX_MAX_SEG - txq->wthresh) {
1054 rte_errno = -EINVAL;
1055 return i;
1056 }
1057
1058 if (ol_flags & TXGBE_TX_OFFLOAD_NOTSUP_MASK) {
1059 rte_errno = -ENOTSUP;
1060 return i;
1061 }
1062
1063#ifdef RTE_LIBRTE_ETHDEV_DEBUG
1064 ret = rte_validate_tx_offload(m);
1065 if (ret != 0) {
1066 rte_errno = ret;
1067 return i;
1068 }
1069#endif
1070 ret = rte_net_intel_cksum_prepare(m);
1071 if (ret != 0) {
1072 rte_errno = ret;
1073 return i;
1074 }
1075 }
1076
1077 return i;
1078}
1079
1080
1081
1082
1083
1084
1085
1086static inline uint32_t
1087txgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask)
1088{
1089 uint16_t ptid = TXGBE_RXD_PTID(pkt_info);
1090
1091 ptid &= ptid_mask;
1092
1093 return txgbe_decode_ptype(ptid);
1094}
1095
1096static inline uint64_t
1097txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info)
1098{
1099 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1100 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH,
1101 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH,
1102 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
1103 0, 0, 0, RTE_MBUF_F_RX_FDIR,
1104 };
1105#ifdef RTE_LIBRTE_IEEE1588
1106 static uint64_t ip_pkt_etqf_map[8] = {
1107 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP,
1108 0, 0, 0, 0,
1109 };
1110 int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info));
1111 if (likely(-1 != etfid))
1112 return ip_pkt_etqf_map[etfid] |
1113 ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1114 else
1115 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1116#else
1117 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1118#endif
1119}
1120
1121static inline uint64_t
1122rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1123{
1124 uint64_t pkt_flags;
1125
1126
1127
1128
1129
1130
1131 pkt_flags = (rx_status & TXGBE_RXD_STAT_VLAN &&
1132 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED)
1133 ? vlan_flags : 0;
1134
1135#ifdef RTE_LIBRTE_IEEE1588
1136 if (rx_status & TXGBE_RXD_STAT_1588)
1137 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST;
1138#endif
1139 return pkt_flags;
1140}
1141
1142static inline uint64_t
1143rx_desc_error_to_pkt_flags(uint32_t rx_status)
1144{
1145 uint64_t pkt_flags = 0;
1146
1147
1148 if (rx_status & TXGBE_RXD_STAT_IPCS) {
1149 pkt_flags |= (rx_status & TXGBE_RXD_ERR_IPCS
1150 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD);
1151 }
1152
1153 if (rx_status & TXGBE_RXD_STAT_L4CS) {
1154 pkt_flags |= (rx_status & TXGBE_RXD_ERR_L4CS
1155 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD);
1156 }
1157
1158 if (rx_status & TXGBE_RXD_STAT_EIPCS &&
1159 rx_status & TXGBE_RXD_ERR_EIPCS) {
1160 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
1161 }
1162
1163#ifdef RTE_LIB_SECURITY
1164 if (rx_status & TXGBE_RXD_STAT_SECP) {
1165 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD;
1166 if (rx_status & TXGBE_RXD_ERR_SECERR)
1167 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED;
1168 }
1169#endif
1170
1171 return pkt_flags;
1172}
1173
1174
1175
1176
1177
1178
1179
1180
1181#define LOOK_AHEAD 8
1182#if (LOOK_AHEAD != 8)
1183#error "PMD TXGBE: LOOK_AHEAD must be 8\n"
1184#endif
1185static inline int
1186txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq)
1187{
1188 volatile struct txgbe_rx_desc *rxdp;
1189 struct txgbe_rx_entry *rxep;
1190 struct rte_mbuf *mb;
1191 uint16_t pkt_len;
1192 uint64_t pkt_flags;
1193 int nb_dd;
1194 uint32_t s[LOOK_AHEAD];
1195 uint32_t pkt_info[LOOK_AHEAD];
1196 int i, j, nb_rx = 0;
1197 uint32_t status;
1198
1199
1200 rxdp = &rxq->rx_ring[rxq->rx_tail];
1201 rxep = &rxq->sw_ring[rxq->rx_tail];
1202
1203 status = rxdp->qw1.lo.status;
1204
1205 if (!(status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1206 return 0;
1207
1208
1209
1210
1211
1212 for (i = 0; i < RTE_PMD_TXGBE_RX_MAX_BURST;
1213 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1214
1215 for (j = 0; j < LOOK_AHEAD; j++)
1216 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status);
1217
1218 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1219
1220
1221 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1222 (s[nb_dd] & TXGBE_RXD_STAT_DD); nb_dd++)
1223 ;
1224
1225 for (j = 0; j < nb_dd; j++)
1226 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0);
1227
1228 nb_rx += nb_dd;
1229
1230
1231 for (j = 0; j < nb_dd; ++j) {
1232 mb = rxep[j].mbuf;
1233 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) -
1234 rxq->crc_len;
1235 mb->data_len = pkt_len;
1236 mb->pkt_len = pkt_len;
1237 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag);
1238
1239
1240 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1241 rxq->vlan_flags);
1242 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1243 pkt_flags |=
1244 txgbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]);
1245 mb->ol_flags = pkt_flags;
1246 mb->packet_type =
1247 txgbe_rxd_pkt_info_to_pkt_type(pkt_info[j],
1248 rxq->pkt_type_mask);
1249
1250 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1251 mb->hash.rss =
1252 rte_le_to_cpu_32(rxdp[j].qw0.dw1);
1253 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1254 mb->hash.fdir.hash =
1255 rte_le_to_cpu_16(rxdp[j].qw0.hi.csum) &
1256 TXGBE_ATR_HASH_MASK;
1257 mb->hash.fdir.id =
1258 rte_le_to_cpu_16(rxdp[j].qw0.hi.ipid);
1259 }
1260 }
1261
1262
1263 for (j = 0; j < LOOK_AHEAD; ++j)
1264 rxq->rx_stage[i + j] = rxep[j].mbuf;
1265
1266
1267 if (nb_dd != LOOK_AHEAD)
1268 break;
1269 }
1270
1271
1272 for (i = 0; i < nb_rx; ++i)
1273 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1274
1275 return nb_rx;
1276}
1277
1278static inline int
1279txgbe_rx_alloc_bufs(struct txgbe_rx_queue *rxq, bool reset_mbuf)
1280{
1281 volatile struct txgbe_rx_desc *rxdp;
1282 struct txgbe_rx_entry *rxep;
1283 struct rte_mbuf *mb;
1284 uint16_t alloc_idx;
1285 __le64 dma_addr;
1286 int diag, i;
1287
1288
1289 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1290 rxep = &rxq->sw_ring[alloc_idx];
1291 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1292 rxq->rx_free_thresh);
1293 if (unlikely(diag != 0))
1294 return -ENOMEM;
1295
1296 rxdp = &rxq->rx_ring[alloc_idx];
1297 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1298
1299 mb = rxep[i].mbuf;
1300 if (reset_mbuf)
1301 mb->port = rxq->port_id;
1302
1303 rte_mbuf_refcnt_set(mb, 1);
1304 mb->data_off = RTE_PKTMBUF_HEADROOM;
1305
1306
1307 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1308 TXGBE_RXD_HDRADDR(&rxdp[i], 0);
1309 TXGBE_RXD_PKTADDR(&rxdp[i], dma_addr);
1310 }
1311
1312
1313 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1314 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1315 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1316
1317
1318 return 0;
1319}
1320
1321static inline uint16_t
1322txgbe_rx_fill_from_stage(struct txgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1323 uint16_t nb_pkts)
1324{
1325 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1326 int i;
1327
1328
1329 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1330
1331
1332 for (i = 0; i < nb_pkts; ++i)
1333 rx_pkts[i] = stage[i];
1334
1335
1336 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1337 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1338
1339 return nb_pkts;
1340}
1341
1342static inline uint16_t
1343txgbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1344 uint16_t nb_pkts)
1345{
1346 struct txgbe_rx_queue *rxq = (struct txgbe_rx_queue *)rx_queue;
1347 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1348 uint16_t nb_rx = 0;
1349
1350
1351 if (rxq->rx_nb_avail)
1352 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1353
1354
1355 nb_rx = (uint16_t)txgbe_rx_scan_hw_ring(rxq);
1356
1357
1358 rxq->rx_next_avail = 0;
1359 rxq->rx_nb_avail = nb_rx;
1360 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1361
1362
1363 if (rxq->rx_tail > rxq->rx_free_trigger) {
1364 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1365
1366 if (txgbe_rx_alloc_bufs(rxq, true) != 0) {
1367 int i, j;
1368
1369 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1370 "queue_id=%u", (uint16_t)rxq->port_id,
1371 (uint16_t)rxq->queue_id);
1372
1373 dev->data->rx_mbuf_alloc_failed +=
1374 rxq->rx_free_thresh;
1375
1376
1377
1378
1379
1380 rxq->rx_nb_avail = 0;
1381 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1382 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1383 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1384
1385 return 0;
1386 }
1387
1388
1389 rte_wmb();
1390 txgbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger);
1391 }
1392
1393 if (rxq->rx_tail >= rxq->nb_rx_desc)
1394 rxq->rx_tail = 0;
1395
1396
1397 if (rxq->rx_nb_avail)
1398 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1399
1400 return 0;
1401}
1402
1403
1404uint16_t
1405txgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1406 uint16_t nb_pkts)
1407{
1408 uint16_t nb_rx;
1409
1410 if (unlikely(nb_pkts == 0))
1411 return 0;
1412
1413 if (likely(nb_pkts <= RTE_PMD_TXGBE_RX_MAX_BURST))
1414 return txgbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1415
1416
1417 nb_rx = 0;
1418 while (nb_pkts) {
1419 uint16_t ret, n;
1420
1421 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_RX_MAX_BURST);
1422 ret = txgbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1423 nb_rx = (uint16_t)(nb_rx + ret);
1424 nb_pkts = (uint16_t)(nb_pkts - ret);
1425 if (ret < n)
1426 break;
1427 }
1428
1429 return nb_rx;
1430}
1431
1432uint16_t
1433txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1434 uint16_t nb_pkts)
1435{
1436 struct txgbe_rx_queue *rxq;
1437 volatile struct txgbe_rx_desc *rx_ring;
1438 volatile struct txgbe_rx_desc *rxdp;
1439 struct txgbe_rx_entry *sw_ring;
1440 struct txgbe_rx_entry *rxe;
1441 struct rte_mbuf *rxm;
1442 struct rte_mbuf *nmb;
1443 struct txgbe_rx_desc rxd;
1444 uint64_t dma_addr;
1445 uint32_t staterr;
1446 uint32_t pkt_info;
1447 uint16_t pkt_len;
1448 uint16_t rx_id;
1449 uint16_t nb_rx;
1450 uint16_t nb_hold;
1451 uint64_t pkt_flags;
1452
1453 nb_rx = 0;
1454 nb_hold = 0;
1455 rxq = rx_queue;
1456 rx_id = rxq->rx_tail;
1457 rx_ring = rxq->rx_ring;
1458 sw_ring = rxq->sw_ring;
1459 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1460 while (nb_rx < nb_pkts) {
1461
1462
1463
1464
1465
1466
1467
1468
1469 rxdp = &rx_ring[rx_id];
1470 staterr = rxdp->qw1.lo.status;
1471 if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1472 break;
1473 rxd = *rxdp;
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1502 "ext_err_stat=0x%08x pkt_len=%u",
1503 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1504 (uint16_t)rx_id, (uint32_t)staterr,
1505 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
1506
1507 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1508 if (nmb == NULL) {
1509 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1510 "queue_id=%u", (uint16_t)rxq->port_id,
1511 (uint16_t)rxq->queue_id);
1512 dev->data->rx_mbuf_alloc_failed++;
1513 break;
1514 }
1515
1516 nb_hold++;
1517 rxe = &sw_ring[rx_id];
1518 rx_id++;
1519 if (rx_id == rxq->nb_rx_desc)
1520 rx_id = 0;
1521
1522
1523 rte_txgbe_prefetch(sw_ring[rx_id].mbuf);
1524
1525
1526
1527
1528
1529
1530 if ((rx_id & 0x3) == 0) {
1531 rte_txgbe_prefetch(&rx_ring[rx_id]);
1532 rte_txgbe_prefetch(&sw_ring[rx_id]);
1533 }
1534
1535 rxm = rxe->mbuf;
1536 rxe->mbuf = nmb;
1537 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1538 TXGBE_RXD_HDRADDR(rxdp, 0);
1539 TXGBE_RXD_PKTADDR(rxdp, dma_addr);
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) -
1555 rxq->crc_len);
1556 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1557 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1558 rxm->nb_segs = 1;
1559 rxm->next = NULL;
1560 rxm->pkt_len = pkt_len;
1561 rxm->data_len = pkt_len;
1562 rxm->port = rxq->port_id;
1563
1564 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0);
1565
1566 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag);
1567
1568 pkt_flags = rx_desc_status_to_pkt_flags(staterr,
1569 rxq->vlan_flags);
1570 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1571 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1572 rxm->ol_flags = pkt_flags;
1573 rxm->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1574 rxq->pkt_type_mask);
1575
1576 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) {
1577 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1);
1578 } else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1579 rxm->hash.fdir.hash =
1580 rte_le_to_cpu_16(rxd.qw0.hi.csum) &
1581 TXGBE_ATR_HASH_MASK;
1582 rxm->hash.fdir.id = rte_le_to_cpu_16(rxd.qw0.hi.ipid);
1583 }
1584
1585
1586
1587
1588 rx_pkts[nb_rx++] = rxm;
1589 }
1590 rxq->rx_tail = rx_id;
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1602 if (nb_hold > rxq->rx_free_thresh) {
1603 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1604 "nb_hold=%u nb_rx=%u",
1605 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1606 (uint16_t)rx_id, (uint16_t)nb_hold,
1607 (uint16_t)nb_rx);
1608 rx_id = (uint16_t)((rx_id == 0) ?
1609 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1610 txgbe_set32(rxq->rdt_reg_addr, rx_id);
1611 nb_hold = 0;
1612 }
1613 rxq->nb_rx_hold = nb_hold;
1614 return nb_rx;
1615}
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631static inline void
1632txgbe_fill_cluster_head_buf(struct rte_mbuf *head, struct txgbe_rx_desc *desc,
1633 struct txgbe_rx_queue *rxq, uint32_t staterr)
1634{
1635 uint32_t pkt_info;
1636 uint64_t pkt_flags;
1637
1638 head->port = rxq->port_id;
1639
1640
1641
1642
1643 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag);
1644 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0);
1645 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1646 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1647 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1648 head->ol_flags = pkt_flags;
1649 head->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1650 rxq->pkt_type_mask);
1651
1652 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) {
1653 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1);
1654 } else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1655 head->hash.fdir.hash = rte_le_to_cpu_16(desc->qw0.hi.csum)
1656 & TXGBE_ATR_HASH_MASK;
1657 head->hash.fdir.id = rte_le_to_cpu_16(desc->qw0.hi.ipid);
1658 }
1659}
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687static inline uint16_t
1688txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
1689 bool bulk_alloc)
1690{
1691 struct txgbe_rx_queue *rxq = rx_queue;
1692 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1693 volatile struct txgbe_rx_desc *rx_ring = rxq->rx_ring;
1694 struct txgbe_rx_entry *sw_ring = rxq->sw_ring;
1695 struct txgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
1696 uint16_t rx_id = rxq->rx_tail;
1697 uint16_t nb_rx = 0;
1698 uint16_t nb_hold = rxq->nb_rx_hold;
1699 uint16_t prev_id = rxq->rx_tail;
1700
1701 while (nb_rx < nb_pkts) {
1702 bool eop;
1703 struct txgbe_rx_entry *rxe;
1704 struct txgbe_scattered_rx_entry *sc_entry;
1705 struct txgbe_scattered_rx_entry *next_sc_entry = NULL;
1706 struct txgbe_rx_entry *next_rxe = NULL;
1707 struct rte_mbuf *first_seg;
1708 struct rte_mbuf *rxm;
1709 struct rte_mbuf *nmb = NULL;
1710 struct txgbe_rx_desc rxd;
1711 uint16_t data_len;
1712 uint16_t next_id;
1713 volatile struct txgbe_rx_desc *rxdp;
1714 uint32_t staterr;
1715
1716next_desc:
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745 rxdp = &rx_ring[rx_id];
1746 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status);
1747
1748 if (!(staterr & TXGBE_RXD_STAT_DD))
1749 break;
1750
1751 rxd = *rxdp;
1752
1753 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1754 "staterr=0x%x data_len=%u",
1755 rxq->port_id, rxq->queue_id, rx_id, staterr,
1756 rte_le_to_cpu_16(rxd.qw1.hi.len));
1757
1758 if (!bulk_alloc) {
1759 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1760 if (nmb == NULL) {
1761 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
1762 "port_id=%u queue_id=%u",
1763 rxq->port_id, rxq->queue_id);
1764
1765 dev->data->rx_mbuf_alloc_failed++;
1766 break;
1767 }
1768 } else if (nb_hold > rxq->rx_free_thresh) {
1769 uint16_t next_rdt = rxq->rx_free_trigger;
1770
1771 if (!txgbe_rx_alloc_bufs(rxq, false)) {
1772 rte_wmb();
1773 txgbe_set32_relaxed(rxq->rdt_reg_addr,
1774 next_rdt);
1775 nb_hold -= rxq->rx_free_thresh;
1776 } else {
1777 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
1778 "port_id=%u queue_id=%u",
1779 rxq->port_id, rxq->queue_id);
1780
1781 dev->data->rx_mbuf_alloc_failed++;
1782 break;
1783 }
1784 }
1785
1786 nb_hold++;
1787 rxe = &sw_ring[rx_id];
1788 eop = staterr & TXGBE_RXD_STAT_EOP;
1789
1790 next_id = rx_id + 1;
1791 if (next_id == rxq->nb_rx_desc)
1792 next_id = 0;
1793
1794
1795 rte_txgbe_prefetch(sw_ring[next_id].mbuf);
1796
1797
1798
1799
1800
1801
1802 if ((next_id & 0x3) == 0) {
1803 rte_txgbe_prefetch(&rx_ring[next_id]);
1804 rte_txgbe_prefetch(&sw_ring[next_id]);
1805 }
1806
1807 rxm = rxe->mbuf;
1808
1809 if (!bulk_alloc) {
1810 __le64 dma =
1811 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1812
1813
1814
1815
1816 rxe->mbuf = nmb;
1817
1818 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1819 TXGBE_RXD_HDRADDR(rxdp, 0);
1820 TXGBE_RXD_PKTADDR(rxdp, dma);
1821 } else {
1822 rxe->mbuf = NULL;
1823 }
1824
1825
1826
1827
1828 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len);
1829 rxm->data_len = data_len;
1830
1831 if (!eop) {
1832 uint16_t nextp_id;
1833
1834
1835
1836
1837
1838
1839 if (TXGBE_RXD_RSCCNT(rxd.qw0.dw0))
1840 nextp_id = TXGBE_RXD_NEXTP(staterr);
1841 else
1842 nextp_id = next_id;
1843
1844 next_sc_entry = &sw_sc_ring[nextp_id];
1845 next_rxe = &sw_ring[nextp_id];
1846 rte_txgbe_prefetch(next_rxe);
1847 }
1848
1849 sc_entry = &sw_sc_ring[rx_id];
1850 first_seg = sc_entry->fbuf;
1851 sc_entry->fbuf = NULL;
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861 if (first_seg == NULL) {
1862 first_seg = rxm;
1863 first_seg->pkt_len = data_len;
1864 first_seg->nb_segs = 1;
1865 } else {
1866 first_seg->pkt_len += data_len;
1867 first_seg->nb_segs++;
1868 }
1869
1870 prev_id = rx_id;
1871 rx_id = next_id;
1872
1873
1874
1875
1876
1877
1878 if (!eop && next_rxe) {
1879 rxm->next = next_rxe->mbuf;
1880 next_sc_entry->fbuf = first_seg;
1881 goto next_desc;
1882 }
1883
1884
1885 txgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
1886
1887
1888
1889
1890
1891
1892 first_seg->pkt_len -= rxq->crc_len;
1893 if (unlikely(rxm->data_len <= rxq->crc_len)) {
1894 struct rte_mbuf *lp;
1895
1896 for (lp = first_seg; lp->next != rxm; lp = lp->next)
1897 ;
1898
1899 first_seg->nb_segs--;
1900 lp->data_len -= rxq->crc_len - rxm->data_len;
1901 lp->next = NULL;
1902 rte_pktmbuf_free_seg(rxm);
1903 } else {
1904 rxm->data_len -= rxq->crc_len;
1905 }
1906
1907
1908 rte_packet_prefetch((char *)first_seg->buf_addr +
1909 first_seg->data_off);
1910
1911
1912
1913
1914
1915 rx_pkts[nb_rx++] = first_seg;
1916 }
1917
1918
1919
1920
1921 rxq->rx_tail = rx_id;
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
1933 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1934 "nb_hold=%u nb_rx=%u",
1935 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
1936
1937 rte_wmb();
1938 txgbe_set32_relaxed(rxq->rdt_reg_addr, prev_id);
1939 nb_hold = 0;
1940 }
1941
1942 rxq->nb_rx_hold = nb_hold;
1943 return nb_rx;
1944}
1945
1946uint16_t
1947txgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1948 uint16_t nb_pkts)
1949{
1950 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
1951}
1952
1953uint16_t
1954txgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1955 uint16_t nb_pkts)
1956{
1957 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
1958}
1959
1960uint64_t
1961txgbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused)
1962{
1963 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
1964}
1965
1966uint64_t
1967txgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
1968{
1969 uint64_t offloads;
1970 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1971 struct rte_eth_dev_sriov *sriov = &RTE_ETH_DEV_SRIOV(dev);
1972
1973 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
1974 RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
1975 RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
1976 RTE_ETH_RX_OFFLOAD_KEEP_CRC |
1977 RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
1978 RTE_ETH_RX_OFFLOAD_RSS_HASH |
1979 RTE_ETH_RX_OFFLOAD_SCATTER;
1980
1981 if (!txgbe_is_vf(dev))
1982 offloads |= (RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
1983 RTE_ETH_RX_OFFLOAD_QINQ_STRIP |
1984 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND);
1985
1986
1987
1988
1989
1990 if (hw->mac.type == txgbe_mac_raptor && !sriov->active)
1991 offloads |= RTE_ETH_RX_OFFLOAD_TCP_LRO;
1992
1993 if (hw->mac.type == txgbe_mac_raptor)
1994 offloads |= RTE_ETH_RX_OFFLOAD_MACSEC_STRIP;
1995
1996 offloads |= RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM;
1997
1998#ifdef RTE_LIB_SECURITY
1999 if (dev->security_ctx)
2000 offloads |= RTE_ETH_RX_OFFLOAD_SECURITY;
2001#endif
2002
2003 return offloads;
2004}
2005
2006static void __rte_cold
2007txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq)
2008{
2009 unsigned int i;
2010
2011 if (txq->sw_ring != NULL) {
2012 for (i = 0; i < txq->nb_tx_desc; i++) {
2013 if (txq->sw_ring[i].mbuf != NULL) {
2014 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2015 txq->sw_ring[i].mbuf = NULL;
2016 }
2017 }
2018 }
2019}
2020
2021static int
2022txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt)
2023{
2024 struct txgbe_tx_entry *swr_ring = txq->sw_ring;
2025 uint16_t i, tx_last, tx_id;
2026 uint16_t nb_tx_free_last;
2027 uint16_t nb_tx_to_clean;
2028 uint32_t pkt_cnt;
2029
2030
2031 tx_last = txq->tx_tail;
2032 tx_id = swr_ring[tx_last].next_id;
2033
2034 if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq))
2035 return 0;
2036
2037 nb_tx_to_clean = txq->nb_tx_free;
2038 nb_tx_free_last = txq->nb_tx_free;
2039 if (!free_cnt)
2040 free_cnt = txq->nb_tx_desc;
2041
2042
2043
2044
2045 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2046 for (i = 0; i < nb_tx_to_clean &&
2047 pkt_cnt < free_cnt &&
2048 tx_id != tx_last; i++) {
2049 if (swr_ring[tx_id].mbuf != NULL) {
2050 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2051 swr_ring[tx_id].mbuf = NULL;
2052
2053
2054
2055
2056
2057 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2058 }
2059
2060 tx_id = swr_ring[tx_id].next_id;
2061 }
2062
2063 if (pkt_cnt < free_cnt) {
2064 if (txgbe_xmit_cleanup(txq))
2065 break;
2066
2067 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2068 nb_tx_free_last = txq->nb_tx_free;
2069 }
2070 }
2071
2072 return (int)pkt_cnt;
2073}
2074
2075static int
2076txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq,
2077 uint32_t free_cnt)
2078{
2079 int i, n, cnt;
2080
2081 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2082 free_cnt = txq->nb_tx_desc;
2083
2084 cnt = free_cnt - free_cnt % txq->tx_free_thresh;
2085
2086 for (i = 0; i < cnt; i += n) {
2087 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh)
2088 break;
2089
2090 n = txgbe_tx_free_bufs(txq);
2091
2092 if (n == 0)
2093 break;
2094 }
2095
2096 return i;
2097}
2098
2099int
2100txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2101{
2102 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
2103 if (txq->offloads == 0 &&
2104#ifdef RTE_LIB_SECURITY
2105 !(txq->using_ipsec) &&
2106#endif
2107 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST)
2108 return txgbe_tx_done_cleanup_simple(txq, free_cnt);
2109
2110 return txgbe_tx_done_cleanup_full(txq, free_cnt);
2111}
2112
2113static void __rte_cold
2114txgbe_tx_free_swring(struct txgbe_tx_queue *txq)
2115{
2116 if (txq != NULL &&
2117 txq->sw_ring != NULL)
2118 rte_free(txq->sw_ring);
2119}
2120
2121static void __rte_cold
2122txgbe_tx_queue_release(struct txgbe_tx_queue *txq)
2123{
2124 if (txq != NULL && txq->ops != NULL) {
2125 txq->ops->release_mbufs(txq);
2126 txq->ops->free_swring(txq);
2127 rte_free(txq);
2128 }
2129}
2130
2131void __rte_cold
2132txgbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2133{
2134 txgbe_tx_queue_release(dev->data->tx_queues[qid]);
2135}
2136
2137
2138static void __rte_cold
2139txgbe_reset_tx_queue(struct txgbe_tx_queue *txq)
2140{
2141 static const struct txgbe_tx_desc zeroed_desc = {0};
2142 struct txgbe_tx_entry *txe = txq->sw_ring;
2143 uint16_t prev, i;
2144
2145
2146 for (i = 0; i < txq->nb_tx_desc; i++)
2147 txq->tx_ring[i] = zeroed_desc;
2148
2149
2150 prev = (uint16_t)(txq->nb_tx_desc - 1);
2151 for (i = 0; i < txq->nb_tx_desc; i++) {
2152 volatile struct txgbe_tx_desc *txd = &txq->tx_ring[i];
2153
2154 txd->dw3 = rte_cpu_to_le_32(TXGBE_TXD_DD);
2155 txe[i].mbuf = NULL;
2156 txe[i].last_id = i;
2157 txe[prev].next_id = i;
2158 prev = i;
2159 }
2160
2161 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
2162 txq->tx_tail = 0;
2163
2164
2165
2166
2167
2168 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2169 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2170 txq->ctx_curr = 0;
2171 memset((void *)&txq->ctx_cache, 0,
2172 TXGBE_CTX_NUM * sizeof(struct txgbe_ctx_info));
2173}
2174
2175static const struct txgbe_txq_ops def_txq_ops = {
2176 .release_mbufs = txgbe_tx_queue_release_mbufs,
2177 .free_swring = txgbe_tx_free_swring,
2178 .reset = txgbe_reset_tx_queue,
2179};
2180
2181
2182
2183
2184
2185void __rte_cold
2186txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq)
2187{
2188
2189 if (txq->offloads == 0 &&
2190#ifdef RTE_LIB_SECURITY
2191 !(txq->using_ipsec) &&
2192#endif
2193 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) {
2194 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2195 dev->tx_pkt_burst = txgbe_xmit_pkts_simple;
2196 dev->tx_pkt_prepare = NULL;
2197 } else {
2198 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2199 PMD_INIT_LOG(DEBUG,
2200 " - offloads = 0x%" PRIx64,
2201 txq->offloads);
2202 PMD_INIT_LOG(DEBUG,
2203 " - tx_free_thresh = %lu [RTE_PMD_TXGBE_TX_MAX_BURST=%lu]",
2204 (unsigned long)txq->tx_free_thresh,
2205 (unsigned long)RTE_PMD_TXGBE_TX_MAX_BURST);
2206 dev->tx_pkt_burst = txgbe_xmit_pkts;
2207 dev->tx_pkt_prepare = txgbe_prep_pkts;
2208 }
2209}
2210
2211uint64_t
2212txgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2213{
2214 RTE_SET_USED(dev);
2215
2216 return 0;
2217}
2218
2219uint64_t
2220txgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2221{
2222 uint64_t tx_offload_capa;
2223
2224 tx_offload_capa =
2225 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
2226 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
2227 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2228 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
2229 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
2230 RTE_ETH_TX_OFFLOAD_TCP_TSO |
2231 RTE_ETH_TX_OFFLOAD_UDP_TSO |
2232 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO |
2233 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO |
2234 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
2235 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO |
2236 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO |
2237 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO |
2238 RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
2239
2240 if (!txgbe_is_vf(dev))
2241 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT;
2242
2243 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_MACSEC_INSERT;
2244
2245 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
2246 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM;
2247
2248#ifdef RTE_LIB_SECURITY
2249 if (dev->security_ctx)
2250 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_SECURITY;
2251#endif
2252 return tx_offload_capa;
2253}
2254
2255int __rte_cold
2256txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2257 uint16_t queue_idx,
2258 uint16_t nb_desc,
2259 unsigned int socket_id,
2260 const struct rte_eth_txconf *tx_conf)
2261{
2262 const struct rte_memzone *tz;
2263 struct txgbe_tx_queue *txq;
2264 struct txgbe_hw *hw;
2265 uint16_t tx_free_thresh;
2266 uint64_t offloads;
2267
2268 PMD_INIT_FUNC_TRACE();
2269 hw = TXGBE_DEV_HW(dev);
2270
2271 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2272
2273
2274
2275
2276
2277
2278 if (nb_desc % TXGBE_TXD_ALIGN != 0 ||
2279 nb_desc > TXGBE_RING_DESC_MAX ||
2280 nb_desc < TXGBE_RING_DESC_MIN) {
2281 return -EINVAL;
2282 }
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2294 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2295 if (tx_free_thresh >= (nb_desc - 3)) {
2296 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the number of "
2297 "TX descriptors minus 3. (tx_free_thresh=%u "
2298 "port=%d queue=%d)",
2299 (unsigned int)tx_free_thresh,
2300 (int)dev->data->port_id, (int)queue_idx);
2301 return -(EINVAL);
2302 }
2303
2304 if ((nb_desc % tx_free_thresh) != 0) {
2305 PMD_INIT_LOG(ERR, "tx_free_thresh must be a divisor of the "
2306 "number of TX descriptors. (tx_free_thresh=%u "
2307 "port=%d queue=%d)", (unsigned int)tx_free_thresh,
2308 (int)dev->data->port_id, (int)queue_idx);
2309 return -(EINVAL);
2310 }
2311
2312
2313 if (dev->data->tx_queues[queue_idx] != NULL) {
2314 txgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2315 dev->data->tx_queues[queue_idx] = NULL;
2316 }
2317
2318
2319 txq = rte_zmalloc_socket("ethdev TX queue",
2320 sizeof(struct txgbe_tx_queue),
2321 RTE_CACHE_LINE_SIZE, socket_id);
2322 if (txq == NULL)
2323 return -ENOMEM;
2324
2325
2326
2327
2328
2329
2330 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2331 sizeof(struct txgbe_tx_desc) * TXGBE_RING_DESC_MAX,
2332 TXGBE_ALIGN, socket_id);
2333 if (tz == NULL) {
2334 txgbe_tx_queue_release(txq);
2335 return -ENOMEM;
2336 }
2337
2338 txq->nb_tx_desc = nb_desc;
2339 txq->tx_free_thresh = tx_free_thresh;
2340 txq->pthresh = tx_conf->tx_thresh.pthresh;
2341 txq->hthresh = tx_conf->tx_thresh.hthresh;
2342 txq->wthresh = tx_conf->tx_thresh.wthresh;
2343 txq->queue_id = queue_idx;
2344 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2345 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2346 txq->port_id = dev->data->port_id;
2347 txq->offloads = offloads;
2348 txq->ops = &def_txq_ops;
2349 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2350#ifdef RTE_LIB_SECURITY
2351 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2352 RTE_ETH_TX_OFFLOAD_SECURITY);
2353#endif
2354
2355
2356
2357
2358 if (hw->mac.type == txgbe_mac_raptor_vf) {
2359 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXWP(queue_idx));
2360 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXCFG(queue_idx));
2361 } else {
2362 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw,
2363 TXGBE_TXWP(txq->reg_idx));
2364 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw,
2365 TXGBE_TXCFG(txq->reg_idx));
2366 }
2367
2368 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
2369 txq->tx_ring = (struct txgbe_tx_desc *)TMZ_VADDR(tz);
2370
2371
2372 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2373 sizeof(struct txgbe_tx_entry) * nb_desc,
2374 RTE_CACHE_LINE_SIZE, socket_id);
2375 if (txq->sw_ring == NULL) {
2376 txgbe_tx_queue_release(txq);
2377 return -ENOMEM;
2378 }
2379 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2380 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2381
2382
2383 txgbe_set_tx_function(dev, txq);
2384
2385 txq->ops->reset(txq);
2386
2387 dev->data->tx_queues[queue_idx] = txq;
2388
2389 return 0;
2390}
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404static void __rte_cold
2405txgbe_free_sc_cluster(struct rte_mbuf *m)
2406{
2407 uint16_t i, nb_segs = m->nb_segs;
2408 struct rte_mbuf *next_seg;
2409
2410 for (i = 0; i < nb_segs; i++) {
2411 next_seg = m->next;
2412 rte_pktmbuf_free_seg(m);
2413 m = next_seg;
2414 }
2415}
2416
2417static void __rte_cold
2418txgbe_rx_queue_release_mbufs(struct txgbe_rx_queue *rxq)
2419{
2420 unsigned int i;
2421
2422 if (rxq->sw_ring != NULL) {
2423 for (i = 0; i < rxq->nb_rx_desc; i++) {
2424 if (rxq->sw_ring[i].mbuf != NULL) {
2425 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2426 rxq->sw_ring[i].mbuf = NULL;
2427 }
2428 }
2429 if (rxq->rx_nb_avail) {
2430 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2431 struct rte_mbuf *mb;
2432
2433 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2434 rte_pktmbuf_free_seg(mb);
2435 }
2436 rxq->rx_nb_avail = 0;
2437 }
2438 }
2439
2440 if (rxq->sw_sc_ring)
2441 for (i = 0; i < rxq->nb_rx_desc; i++)
2442 if (rxq->sw_sc_ring[i].fbuf) {
2443 txgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2444 rxq->sw_sc_ring[i].fbuf = NULL;
2445 }
2446}
2447
2448static void __rte_cold
2449txgbe_rx_queue_release(struct txgbe_rx_queue *rxq)
2450{
2451 if (rxq != NULL) {
2452 txgbe_rx_queue_release_mbufs(rxq);
2453 rte_free(rxq->sw_ring);
2454 rte_free(rxq->sw_sc_ring);
2455 rte_free(rxq);
2456 }
2457}
2458
2459void __rte_cold
2460txgbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2461{
2462 txgbe_rx_queue_release(dev->data->rx_queues[qid]);
2463}
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473static inline int __rte_cold
2474check_rx_burst_bulk_alloc_preconditions(struct txgbe_rx_queue *rxq)
2475{
2476 int ret = 0;
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486 if (!(rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST)) {
2487 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2488 "rxq->rx_free_thresh=%d, "
2489 "RTE_PMD_TXGBE_RX_MAX_BURST=%d",
2490 rxq->rx_free_thresh, RTE_PMD_TXGBE_RX_MAX_BURST);
2491 ret = -EINVAL;
2492 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2493 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2494 "rxq->rx_free_thresh=%d, "
2495 "rxq->nb_rx_desc=%d",
2496 rxq->rx_free_thresh, rxq->nb_rx_desc);
2497 ret = -EINVAL;
2498 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2499 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2500 "rxq->nb_rx_desc=%d, "
2501 "rxq->rx_free_thresh=%d",
2502 rxq->nb_rx_desc, rxq->rx_free_thresh);
2503 ret = -EINVAL;
2504 }
2505
2506 return ret;
2507}
2508
2509
2510static void __rte_cold
2511txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq)
2512{
2513 static const struct txgbe_rx_desc zeroed_desc = {
2514 {{0}, {0} }, {{0}, {0} } };
2515 unsigned int i;
2516 uint16_t len = rxq->nb_rx_desc;
2517
2518
2519
2520
2521
2522
2523 if (adapter->rx_bulk_alloc_allowed)
2524
2525 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2526
2527
2528
2529
2530
2531
2532 for (i = 0; i < len; i++)
2533 rxq->rx_ring[i] = zeroed_desc;
2534
2535
2536
2537
2538
2539 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2540 for (i = rxq->nb_rx_desc; i < len; ++i)
2541 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2542
2543 rxq->rx_nb_avail = 0;
2544 rxq->rx_next_avail = 0;
2545 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2546 rxq->rx_tail = 0;
2547 rxq->nb_rx_hold = 0;
2548 rxq->pkt_first_seg = NULL;
2549 rxq->pkt_last_seg = NULL;
2550}
2551
2552int __rte_cold
2553txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2554 uint16_t queue_idx,
2555 uint16_t nb_desc,
2556 unsigned int socket_id,
2557 const struct rte_eth_rxconf *rx_conf,
2558 struct rte_mempool *mp)
2559{
2560 const struct rte_memzone *rz;
2561 struct txgbe_rx_queue *rxq;
2562 struct txgbe_hw *hw;
2563 uint16_t len;
2564 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2565 uint64_t offloads;
2566
2567 PMD_INIT_FUNC_TRACE();
2568 hw = TXGBE_DEV_HW(dev);
2569
2570 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
2571
2572
2573
2574
2575
2576
2577 if (nb_desc % TXGBE_RXD_ALIGN != 0 ||
2578 nb_desc > TXGBE_RING_DESC_MAX ||
2579 nb_desc < TXGBE_RING_DESC_MIN) {
2580 return -EINVAL;
2581 }
2582
2583
2584 if (dev->data->rx_queues[queue_idx] != NULL) {
2585 txgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2586 dev->data->rx_queues[queue_idx] = NULL;
2587 }
2588
2589
2590 rxq = rte_zmalloc_socket("ethdev RX queue",
2591 sizeof(struct txgbe_rx_queue),
2592 RTE_CACHE_LINE_SIZE, socket_id);
2593 if (rxq == NULL)
2594 return -ENOMEM;
2595 rxq->mb_pool = mp;
2596 rxq->nb_rx_desc = nb_desc;
2597 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2598 rxq->queue_id = queue_idx;
2599 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2600 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2601 rxq->port_id = dev->data->port_id;
2602 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2603 rxq->crc_len = RTE_ETHER_CRC_LEN;
2604 else
2605 rxq->crc_len = 0;
2606 rxq->drop_en = rx_conf->rx_drop_en;
2607 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2608 rxq->offloads = offloads;
2609
2610
2611
2612
2613
2614 rxq->pkt_type_mask = TXGBE_PTID_MASK;
2615
2616
2617
2618
2619
2620
2621 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2622 RX_RING_SZ, TXGBE_ALIGN, socket_id);
2623 if (rz == NULL) {
2624 txgbe_rx_queue_release(rxq);
2625 return -ENOMEM;
2626 }
2627
2628
2629
2630
2631 memset(rz->addr, 0, RX_RING_SZ);
2632
2633
2634
2635
2636 if (hw->mac.type == txgbe_mac_raptor_vf) {
2637 rxq->rdt_reg_addr =
2638 TXGBE_REG_ADDR(hw, TXGBE_RXWP(queue_idx));
2639 rxq->rdh_reg_addr =
2640 TXGBE_REG_ADDR(hw, TXGBE_RXRP(queue_idx));
2641 } else {
2642 rxq->rdt_reg_addr =
2643 TXGBE_REG_ADDR(hw, TXGBE_RXWP(rxq->reg_idx));
2644 rxq->rdh_reg_addr =
2645 TXGBE_REG_ADDR(hw, TXGBE_RXRP(rxq->reg_idx));
2646 }
2647
2648 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
2649 rxq->rx_ring = (struct txgbe_rx_desc *)TMZ_VADDR(rz);
2650
2651
2652
2653
2654
2655
2656 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2657 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
2658 "preconditions - canceling the feature for "
2659 "the whole port[%d]",
2660 rxq->queue_id, rxq->port_id);
2661 adapter->rx_bulk_alloc_allowed = false;
2662 }
2663
2664
2665
2666
2667
2668
2669 len = nb_desc;
2670 if (adapter->rx_bulk_alloc_allowed)
2671 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2672
2673 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2674 sizeof(struct txgbe_rx_entry) * len,
2675 RTE_CACHE_LINE_SIZE, socket_id);
2676 if (!rxq->sw_ring) {
2677 txgbe_rx_queue_release(rxq);
2678 return -ENOMEM;
2679 }
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689 rxq->sw_sc_ring =
2690 rte_zmalloc_socket("rxq->sw_sc_ring",
2691 sizeof(struct txgbe_scattered_rx_entry) * len,
2692 RTE_CACHE_LINE_SIZE, socket_id);
2693 if (!rxq->sw_sc_ring) {
2694 txgbe_rx_queue_release(rxq);
2695 return -ENOMEM;
2696 }
2697
2698 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
2699 "dma_addr=0x%" PRIx64,
2700 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
2701 rxq->rx_ring_phys_addr);
2702
2703 dev->data->rx_queues[queue_idx] = rxq;
2704
2705 txgbe_reset_rx_queue(adapter, rxq);
2706
2707 return 0;
2708}
2709
2710uint32_t
2711txgbe_dev_rx_queue_count(void *rx_queue)
2712{
2713#define TXGBE_RXQ_SCAN_INTERVAL 4
2714 volatile struct txgbe_rx_desc *rxdp;
2715 struct txgbe_rx_queue *rxq;
2716 uint32_t desc = 0;
2717
2718 rxq = rx_queue;
2719 rxdp = &rxq->rx_ring[rxq->rx_tail];
2720
2721 while ((desc < rxq->nb_rx_desc) &&
2722 (rxdp->qw1.lo.status &
2723 rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) {
2724 desc += TXGBE_RXQ_SCAN_INTERVAL;
2725 rxdp += TXGBE_RXQ_SCAN_INTERVAL;
2726 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2727 rxdp = &(rxq->rx_ring[rxq->rx_tail +
2728 desc - rxq->nb_rx_desc]);
2729 }
2730
2731 return desc;
2732}
2733
2734int
2735txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
2736{
2737 struct txgbe_rx_queue *rxq = rx_queue;
2738 volatile uint32_t *status;
2739 uint32_t nb_hold, desc;
2740
2741 if (unlikely(offset >= rxq->nb_rx_desc))
2742 return -EINVAL;
2743
2744 nb_hold = rxq->nb_rx_hold;
2745 if (offset >= rxq->nb_rx_desc - nb_hold)
2746 return RTE_ETH_RX_DESC_UNAVAIL;
2747
2748 desc = rxq->rx_tail + offset;
2749 if (desc >= rxq->nb_rx_desc)
2750 desc -= rxq->nb_rx_desc;
2751
2752 status = &rxq->rx_ring[desc].qw1.lo.status;
2753 if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))
2754 return RTE_ETH_RX_DESC_DONE;
2755
2756 return RTE_ETH_RX_DESC_AVAIL;
2757}
2758
2759int
2760txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
2761{
2762 struct txgbe_tx_queue *txq = tx_queue;
2763 volatile uint32_t *status;
2764 uint32_t desc;
2765
2766 if (unlikely(offset >= txq->nb_tx_desc))
2767 return -EINVAL;
2768
2769 desc = txq->tx_tail + offset;
2770 if (desc >= txq->nb_tx_desc) {
2771 desc -= txq->nb_tx_desc;
2772 if (desc >= txq->nb_tx_desc)
2773 desc -= txq->nb_tx_desc;
2774 }
2775
2776 status = &txq->tx_ring[desc].dw3;
2777 if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD))
2778 return RTE_ETH_TX_DESC_DONE;
2779
2780 return RTE_ETH_TX_DESC_FULL;
2781}
2782
2783void __rte_cold
2784txgbe_dev_clear_queues(struct rte_eth_dev *dev)
2785{
2786 unsigned int i;
2787 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2788
2789 PMD_INIT_FUNC_TRACE();
2790
2791 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2792 struct txgbe_tx_queue *txq = dev->data->tx_queues[i];
2793
2794 if (txq != NULL) {
2795 txq->ops->release_mbufs(txq);
2796 txq->ops->reset(txq);
2797 }
2798 }
2799
2800 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2801 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
2802
2803 if (rxq != NULL) {
2804 txgbe_rx_queue_release_mbufs(rxq);
2805 txgbe_reset_rx_queue(adapter, rxq);
2806 }
2807 }
2808}
2809
2810void
2811txgbe_dev_free_queues(struct rte_eth_dev *dev)
2812{
2813 unsigned int i;
2814
2815 PMD_INIT_FUNC_TRACE();
2816
2817 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2818 txgbe_dev_rx_queue_release(dev, i);
2819 dev->data->rx_queues[i] = NULL;
2820 }
2821 dev->data->nb_rx_queues = 0;
2822
2823 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2824 txgbe_dev_tx_queue_release(dev, i);
2825 dev->data->tx_queues[i] = NULL;
2826 }
2827 dev->data->nb_tx_queues = 0;
2828}
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849static uint8_t rss_intel_key[40] = {
2850 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2851 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2852 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2853 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2854 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2855};
2856
2857static void
2858txgbe_rss_disable(struct rte_eth_dev *dev)
2859{
2860 struct txgbe_hw *hw;
2861
2862 hw = TXGBE_DEV_HW(dev);
2863 if (hw->mac.type == txgbe_mac_raptor_vf)
2864 wr32m(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_RSSENA, 0);
2865 else
2866 wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0);
2867}
2868
2869int
2870txgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2871 struct rte_eth_rss_conf *rss_conf)
2872{
2873 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2874 uint8_t *hash_key;
2875 uint32_t mrqc;
2876 uint32_t rss_key;
2877 uint64_t rss_hf;
2878 uint16_t i;
2879
2880 if (!txgbe_rss_update_sp(hw->mac.type)) {
2881 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
2882 "NIC.");
2883 return -ENOTSUP;
2884 }
2885
2886 hash_key = rss_conf->rss_key;
2887 if (hash_key) {
2888
2889 for (i = 0; i < 10; i++) {
2890 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF);
2891 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF);
2892 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF);
2893 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF);
2894 wr32at(hw, TXGBE_REG_RSSKEY, i, rss_key);
2895 }
2896 }
2897
2898
2899 rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL;
2900 if (hw->mac.type == txgbe_mac_raptor_vf) {
2901 mrqc = rd32(hw, TXGBE_VFPLCFG);
2902 mrqc &= ~TXGBE_VFPLCFG_RSSMASK;
2903 if (rss_hf & RTE_ETH_RSS_IPV4)
2904 mrqc |= TXGBE_VFPLCFG_RSSIPV4;
2905 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
2906 mrqc |= TXGBE_VFPLCFG_RSSIPV4TCP;
2907 if (rss_hf & RTE_ETH_RSS_IPV6 ||
2908 rss_hf & RTE_ETH_RSS_IPV6_EX)
2909 mrqc |= TXGBE_VFPLCFG_RSSIPV6;
2910 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP ||
2911 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX)
2912 mrqc |= TXGBE_VFPLCFG_RSSIPV6TCP;
2913 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
2914 mrqc |= TXGBE_VFPLCFG_RSSIPV4UDP;
2915 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP ||
2916 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX)
2917 mrqc |= TXGBE_VFPLCFG_RSSIPV6UDP;
2918
2919 if (rss_hf)
2920 mrqc |= TXGBE_VFPLCFG_RSSENA;
2921 else
2922 mrqc &= ~TXGBE_VFPLCFG_RSSENA;
2923
2924 if (dev->data->nb_rx_queues > 3)
2925 mrqc |= TXGBE_VFPLCFG_RSSHASH(2);
2926 else if (dev->data->nb_rx_queues > 1)
2927 mrqc |= TXGBE_VFPLCFG_RSSHASH(1);
2928
2929 wr32(hw, TXGBE_VFPLCFG, mrqc);
2930 } else {
2931 mrqc = rd32(hw, TXGBE_RACTL);
2932 mrqc &= ~TXGBE_RACTL_RSSMASK;
2933 if (rss_hf & RTE_ETH_RSS_IPV4)
2934 mrqc |= TXGBE_RACTL_RSSIPV4;
2935 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
2936 mrqc |= TXGBE_RACTL_RSSIPV4TCP;
2937 if (rss_hf & RTE_ETH_RSS_IPV6 ||
2938 rss_hf & RTE_ETH_RSS_IPV6_EX)
2939 mrqc |= TXGBE_RACTL_RSSIPV6;
2940 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP ||
2941 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX)
2942 mrqc |= TXGBE_RACTL_RSSIPV6TCP;
2943 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
2944 mrqc |= TXGBE_RACTL_RSSIPV4UDP;
2945 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP ||
2946 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX)
2947 mrqc |= TXGBE_RACTL_RSSIPV6UDP;
2948
2949 if (rss_hf)
2950 mrqc |= TXGBE_RACTL_RSSENA;
2951 else
2952 mrqc &= ~TXGBE_RACTL_RSSENA;
2953
2954 wr32(hw, TXGBE_RACTL, mrqc);
2955 }
2956
2957 return 0;
2958}
2959
2960int
2961txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2962 struct rte_eth_rss_conf *rss_conf)
2963{
2964 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2965 uint8_t *hash_key;
2966 uint32_t mrqc;
2967 uint32_t rss_key;
2968 uint64_t rss_hf;
2969 uint16_t i;
2970
2971 hash_key = rss_conf->rss_key;
2972 if (hash_key) {
2973
2974 for (i = 0; i < 10; i++) {
2975 rss_key = rd32at(hw, TXGBE_REG_RSSKEY, i);
2976 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF);
2977 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF);
2978 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF);
2979 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF);
2980 }
2981 }
2982
2983 rss_hf = 0;
2984 if (hw->mac.type == txgbe_mac_raptor_vf) {
2985 mrqc = rd32(hw, TXGBE_VFPLCFG);
2986 if (mrqc & TXGBE_VFPLCFG_RSSIPV4)
2987 rss_hf |= RTE_ETH_RSS_IPV4;
2988 if (mrqc & TXGBE_VFPLCFG_RSSIPV4TCP)
2989 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
2990 if (mrqc & TXGBE_VFPLCFG_RSSIPV6)
2991 rss_hf |= RTE_ETH_RSS_IPV6 |
2992 RTE_ETH_RSS_IPV6_EX;
2993 if (mrqc & TXGBE_VFPLCFG_RSSIPV6TCP)
2994 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP |
2995 RTE_ETH_RSS_IPV6_TCP_EX;
2996 if (mrqc & TXGBE_VFPLCFG_RSSIPV4UDP)
2997 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
2998 if (mrqc & TXGBE_VFPLCFG_RSSIPV6UDP)
2999 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP |
3000 RTE_ETH_RSS_IPV6_UDP_EX;
3001 if (!(mrqc & TXGBE_VFPLCFG_RSSENA))
3002 rss_hf = 0;
3003 } else {
3004 mrqc = rd32(hw, TXGBE_RACTL);
3005 if (mrqc & TXGBE_RACTL_RSSIPV4)
3006 rss_hf |= RTE_ETH_RSS_IPV4;
3007 if (mrqc & TXGBE_RACTL_RSSIPV4TCP)
3008 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
3009 if (mrqc & TXGBE_RACTL_RSSIPV6)
3010 rss_hf |= RTE_ETH_RSS_IPV6 |
3011 RTE_ETH_RSS_IPV6_EX;
3012 if (mrqc & TXGBE_RACTL_RSSIPV6TCP)
3013 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP |
3014 RTE_ETH_RSS_IPV6_TCP_EX;
3015 if (mrqc & TXGBE_RACTL_RSSIPV4UDP)
3016 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
3017 if (mrqc & TXGBE_RACTL_RSSIPV6UDP)
3018 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP |
3019 RTE_ETH_RSS_IPV6_UDP_EX;
3020 if (!(mrqc & TXGBE_RACTL_RSSENA))
3021 rss_hf = 0;
3022 }
3023
3024 rss_hf &= TXGBE_RSS_OFFLOAD_ALL;
3025
3026 rss_conf->rss_hf = rss_hf;
3027 return 0;
3028}
3029
3030static void
3031txgbe_rss_configure(struct rte_eth_dev *dev)
3032{
3033 struct rte_eth_rss_conf rss_conf;
3034 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
3035 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3036 uint32_t reta;
3037 uint16_t i;
3038 uint16_t j;
3039
3040 PMD_INIT_FUNC_TRACE();
3041
3042
3043
3044
3045
3046
3047 if (adapter->rss_reta_updated == 0) {
3048 reta = 0;
3049 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) {
3050 if (j == dev->data->nb_rx_queues)
3051 j = 0;
3052 reta = (reta >> 8) | LS32(j, 24, 0xFF);
3053 if ((i & 3) == 3)
3054 wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
3055 }
3056 }
3057
3058
3059
3060
3061 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
3062 if (rss_conf.rss_key == NULL)
3063 rss_conf.rss_key = rss_intel_key;
3064 txgbe_dev_rss_hash_update(dev, &rss_conf);
3065}
3066
3067#define NUM_VFTA_REGISTERS 128
3068#define NIC_RX_BUFFER_SIZE 0x200
3069
3070static void
3071txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
3072{
3073 struct rte_eth_vmdq_dcb_conf *cfg;
3074 struct txgbe_hw *hw;
3075 enum rte_eth_nb_pools num_pools;
3076 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3077 uint16_t pbsize;
3078 uint8_t nb_tcs;
3079 int i;
3080
3081 PMD_INIT_FUNC_TRACE();
3082 hw = TXGBE_DEV_HW(dev);
3083 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3084 num_pools = cfg->nb_queue_pools;
3085
3086 if (num_pools != RTE_ETH_16_POOLS && num_pools != RTE_ETH_32_POOLS) {
3087 txgbe_rss_disable(dev);
3088 return;
3089 }
3090
3091 nb_tcs = (uint8_t)(RTE_ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3092
3093
3094
3095
3096 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3097 for (i = 0; i < nb_tcs; i++) {
3098 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
3099
3100 rxpbsize &= (~(0x3FF << 10));
3101
3102 rxpbsize |= (pbsize << 10);
3103 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3104 }
3105
3106 for (i = nb_tcs; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3107 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
3108
3109 rxpbsize &= (~(0x3FF << 10));
3110
3111 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3112 }
3113
3114 if (num_pools == RTE_ETH_16_POOLS) {
3115 mrqc = TXGBE_PORTCTL_NUMTC_8;
3116 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3117 } else {
3118 mrqc = TXGBE_PORTCTL_NUMTC_4;
3119 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3120 }
3121 wr32m(hw, TXGBE_PORTCTL,
3122 TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3123
3124 vt_ctl = TXGBE_POOLCTL_RPLEN;
3125 if (cfg->enable_default_pool)
3126 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3127 else
3128 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3129
3130 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3131
3132 queue_mapping = 0;
3133 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++)
3134
3135
3136
3137
3138 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3139
3140 wr32(hw, TXGBE_RPUP2TC, queue_mapping);
3141
3142 wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM);
3143
3144
3145 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3146 vlanctrl |= TXGBE_VLANCTL_VFE;
3147 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3148
3149
3150 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3151 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3152
3153 wr32(hw, TXGBE_POOLRXENA(0),
3154 num_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3155
3156 wr32(hw, TXGBE_ETHADDRIDX, 0);
3157 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3158 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3159
3160
3161 for (i = 0; i < cfg->nb_pool_maps; i++) {
3162
3163 wr32(hw, TXGBE_PSRVLANIDX, i);
3164 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3165 (cfg->pool_map[i].vlan_id & 0xFFF)));
3166
3167 wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools);
3168 }
3169}
3170
3171
3172
3173
3174
3175
3176static void
3177txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3178 struct txgbe_dcb_config *dcb_config)
3179{
3180 uint32_t reg;
3181 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3182
3183 PMD_INIT_FUNC_TRACE();
3184
3185
3186 reg = rd32(hw, TXGBE_ARBTXCTL);
3187 reg |= TXGBE_ARBTXCTL_DIA;
3188 wr32(hw, TXGBE_ARBTXCTL, reg);
3189
3190
3191 reg = rd32(hw, TXGBE_PORTCTL);
3192 reg &= TXGBE_PORTCTL_NUMTC_MASK;
3193 reg |= TXGBE_PORTCTL_DCB;
3194 if (dcb_config->num_tcs.pg_tcs == 8)
3195 reg |= TXGBE_PORTCTL_NUMTC_8;
3196 else
3197 reg |= TXGBE_PORTCTL_NUMTC_4;
3198
3199 wr32(hw, TXGBE_PORTCTL, reg);
3200
3201
3202 reg = rd32(hw, TXGBE_ARBTXCTL);
3203 reg &= ~TXGBE_ARBTXCTL_DIA;
3204 wr32(hw, TXGBE_ARBTXCTL, reg);
3205}
3206
3207
3208
3209
3210
3211
3212static void
3213txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3214 struct txgbe_dcb_config *dcb_config)
3215{
3216 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3217 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3218 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3219
3220 PMD_INIT_FUNC_TRACE();
3221
3222 wr32(hw, TXGBE_POOLTXENA(0),
3223 vmdq_tx_conf->nb_queue_pools ==
3224 RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3225
3226
3227 txgbe_dcb_tx_hw_config(dev, dcb_config);
3228}
3229
3230static void
3231txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3232 struct txgbe_dcb_config *dcb_config)
3233{
3234 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3235 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3236 struct txgbe_dcb_tc_config *tc;
3237 uint8_t i, j;
3238
3239
3240 if (vmdq_rx_conf->nb_queue_pools == RTE_ETH_16_POOLS) {
3241 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS;
3242 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS;
3243 } else {
3244 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS;
3245 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS;
3246 }
3247
3248
3249 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3250 tc = &dcb_config->tc_config[j];
3251 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3252 }
3253
3254
3255 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3256 j = vmdq_rx_conf->dcb_tc[i];
3257 tc = &dcb_config->tc_config[j];
3258 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3259 (uint8_t)(1 << i);
3260 }
3261}
3262
3263static void
3264txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3265 struct txgbe_dcb_config *dcb_config)
3266{
3267 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3268 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3269 struct txgbe_dcb_tc_config *tc;
3270 uint8_t i, j;
3271
3272
3273 if (vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS) {
3274 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS;
3275 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS;
3276 } else {
3277 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS;
3278 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS;
3279 }
3280
3281
3282 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3283 tc = &dcb_config->tc_config[j];
3284 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3285 }
3286
3287
3288 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3289 j = vmdq_tx_conf->dcb_tc[i];
3290 tc = &dcb_config->tc_config[j];
3291 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3292 (uint8_t)(1 << i);
3293 }
3294}
3295
3296static void
3297txgbe_dcb_rx_config(struct rte_eth_dev *dev,
3298 struct txgbe_dcb_config *dcb_config)
3299{
3300 struct rte_eth_dcb_rx_conf *rx_conf =
3301 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3302 struct txgbe_dcb_tc_config *tc;
3303 uint8_t i, j;
3304
3305 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3306 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3307
3308
3309 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3310 tc = &dcb_config->tc_config[j];
3311 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3312 }
3313
3314
3315 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3316 j = rx_conf->dcb_tc[i];
3317 tc = &dcb_config->tc_config[j];
3318 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3319 (uint8_t)(1 << i);
3320 }
3321}
3322
3323static void
3324txgbe_dcb_tx_config(struct rte_eth_dev *dev,
3325 struct txgbe_dcb_config *dcb_config)
3326{
3327 struct rte_eth_dcb_tx_conf *tx_conf =
3328 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3329 struct txgbe_dcb_tc_config *tc;
3330 uint8_t i, j;
3331
3332 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3333 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3334
3335
3336 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3337 tc = &dcb_config->tc_config[j];
3338 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3339 }
3340
3341
3342 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3343 j = tx_conf->dcb_tc[i];
3344 tc = &dcb_config->tc_config[j];
3345 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3346 (uint8_t)(1 << i);
3347 }
3348}
3349
3350
3351
3352
3353
3354
3355static void
3356txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3357 struct txgbe_dcb_config *dcb_config)
3358{
3359 uint32_t reg;
3360 uint32_t vlanctrl;
3361 uint8_t i;
3362 uint32_t q;
3363 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3364
3365 PMD_INIT_FUNC_TRACE();
3366
3367
3368
3369
3370 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA;
3371 wr32(hw, TXGBE_ARBRXCTL, reg);
3372
3373 reg = rd32(hw, TXGBE_PORTCTL);
3374 reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3375 if (dcb_config->num_tcs.pg_tcs == 4) {
3376 reg |= TXGBE_PORTCTL_NUMTC_4;
3377 if (dcb_config->vt_mode)
3378 reg |= TXGBE_PORTCTL_NUMVT_32;
3379 else
3380 wr32(hw, TXGBE_POOLCTL, 0);
3381 }
3382
3383 if (dcb_config->num_tcs.pg_tcs == 8) {
3384 reg |= TXGBE_PORTCTL_NUMTC_8;
3385 if (dcb_config->vt_mode)
3386 reg |= TXGBE_PORTCTL_NUMVT_16;
3387 else
3388 wr32(hw, TXGBE_POOLCTL, 0);
3389 }
3390
3391 wr32(hw, TXGBE_PORTCTL, reg);
3392
3393 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3394
3395 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3396 u32 val = 1 << (q % 32);
3397 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3398 }
3399 } else {
3400
3401 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3402 u32 val = 1 << (q % 32);
3403 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3404 }
3405 }
3406
3407
3408 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3409 vlanctrl |= TXGBE_VLANCTL_VFE;
3410 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3411
3412
3413 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3414 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3415
3416
3417
3418
3419
3420 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP;
3421 wr32(hw, TXGBE_ARBRXCTL, reg);
3422}
3423
3424static void
3425txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill,
3426 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3427{
3428 txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id,
3429 tsa, map);
3430}
3431
3432static void
3433txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill,
3434 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3435{
3436 switch (hw->mac.type) {
3437 case txgbe_mac_raptor:
3438 txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill,
3439 max, bwg_id, tsa);
3440 txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill,
3441 max, bwg_id, tsa, map);
3442 break;
3443 default:
3444 break;
3445 }
3446}
3447
3448#define DCB_RX_CONFIG 1
3449#define DCB_TX_CONFIG 1
3450#define DCB_TX_PB 1024
3451
3452
3453
3454
3455
3456
3457static int
3458txgbe_dcb_hw_configure(struct rte_eth_dev *dev,
3459 struct txgbe_dcb_config *dcb_config)
3460{
3461 int ret = 0;
3462 uint8_t i, pfc_en, nb_tcs;
3463 uint16_t pbsize, rx_buffer_size;
3464 uint8_t config_dcb_rx = 0;
3465 uint8_t config_dcb_tx = 0;
3466 uint8_t tsa[TXGBE_DCB_TC_MAX] = {0};
3467 uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0};
3468 uint16_t refill[TXGBE_DCB_TC_MAX] = {0};
3469 uint16_t max[TXGBE_DCB_TC_MAX] = {0};
3470 uint8_t map[TXGBE_DCB_TC_MAX] = {0};
3471 struct txgbe_dcb_tc_config *tc;
3472 uint32_t max_frame = dev->data->mtu +
3473 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3474 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3475 struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev);
3476
3477 switch (dev->data->dev_conf.rxmode.mq_mode) {
3478 case RTE_ETH_MQ_RX_VMDQ_DCB:
3479 dcb_config->vt_mode = true;
3480 config_dcb_rx = DCB_RX_CONFIG;
3481
3482
3483
3484
3485 txgbe_vmdq_dcb_rx_config(dev, dcb_config);
3486
3487 txgbe_vmdq_dcb_configure(dev);
3488 break;
3489 case RTE_ETH_MQ_RX_DCB:
3490 case RTE_ETH_MQ_RX_DCB_RSS:
3491 dcb_config->vt_mode = false;
3492 config_dcb_rx = DCB_RX_CONFIG;
3493
3494 txgbe_dcb_rx_config(dev, dcb_config);
3495
3496 txgbe_dcb_rx_hw_config(dev, dcb_config);
3497 break;
3498 default:
3499 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
3500 break;
3501 }
3502 switch (dev->data->dev_conf.txmode.mq_mode) {
3503 case RTE_ETH_MQ_TX_VMDQ_DCB:
3504 dcb_config->vt_mode = true;
3505 config_dcb_tx = DCB_TX_CONFIG;
3506
3507
3508
3509 txgbe_dcb_vt_tx_config(dev, dcb_config);
3510
3511 txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
3512 break;
3513
3514 case RTE_ETH_MQ_TX_DCB:
3515 dcb_config->vt_mode = false;
3516 config_dcb_tx = DCB_TX_CONFIG;
3517
3518 txgbe_dcb_tx_config(dev, dcb_config);
3519
3520 txgbe_dcb_tx_hw_config(dev, dcb_config);
3521 break;
3522 default:
3523 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
3524 break;
3525 }
3526
3527 nb_tcs = dcb_config->num_tcs.pfc_tcs;
3528
3529 txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map);
3530 if (nb_tcs == RTE_ETH_4_TCS) {
3531
3532 uint8_t j = 4;
3533 uint8_t mask = 0xFF;
3534
3535 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
3536 mask = (uint8_t)(mask & (~(1 << map[i])));
3537 for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) {
3538 if ((mask & 0x1) && j < RTE_ETH_DCB_NUM_USER_PRIORITIES)
3539 map[j++] = i;
3540 mask >>= 1;
3541 }
3542
3543 for (i = 0; i < nb_tcs; i++) {
3544 tc = &dcb_config->tc_config[i];
3545 if (bw_conf->tc_num != nb_tcs)
3546 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3547 (uint8_t)(100 / nb_tcs);
3548 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3549 (uint8_t)(100 / nb_tcs);
3550 }
3551 for (; i < TXGBE_DCB_TC_MAX; i++) {
3552 tc = &dcb_config->tc_config[i];
3553 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0;
3554 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0;
3555 }
3556 } else {
3557
3558 for (i = 0; i < nb_tcs; i++) {
3559 tc = &dcb_config->tc_config[i];
3560 if (bw_conf->tc_num != nb_tcs)
3561 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3562 (uint8_t)(100 / nb_tcs + (i & 1));
3563 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3564 (uint8_t)(100 / nb_tcs + (i & 1));
3565 }
3566 }
3567
3568 rx_buffer_size = NIC_RX_BUFFER_SIZE;
3569
3570 if (config_dcb_rx) {
3571
3572 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3573 uint32_t rxpbsize = pbsize << 10;
3574
3575 for (i = 0; i < nb_tcs; i++)
3576 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3577
3578
3579 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++)
3580 wr32(hw, TXGBE_PBRXSIZE(i), 0);
3581 }
3582 if (config_dcb_tx) {
3583
3584
3585
3586 uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs;
3587 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) -
3588 TXGBE_TXPKT_SIZE_MAX;
3589
3590 for (i = 0; i < nb_tcs; i++) {
3591 wr32(hw, TXGBE_PBTXSIZE(i), txpktsize);
3592 wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh);
3593 }
3594
3595 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3596 wr32(hw, TXGBE_PBTXSIZE(i), 0);
3597 wr32(hw, TXGBE_PBTXDMATH(i), 0);
3598 }
3599 }
3600
3601
3602 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3603 TXGBE_DCB_TX_CONFIG);
3604 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3605 TXGBE_DCB_RX_CONFIG);
3606
3607 if (config_dcb_rx) {
3608
3609 txgbe_dcb_unpack_refill_cee(dcb_config,
3610 TXGBE_DCB_RX_CONFIG, refill);
3611 txgbe_dcb_unpack_max_cee(dcb_config, max);
3612 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3613 TXGBE_DCB_RX_CONFIG, bwgid);
3614 txgbe_dcb_unpack_tsa_cee(dcb_config,
3615 TXGBE_DCB_RX_CONFIG, tsa);
3616
3617 txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
3618 }
3619
3620 if (config_dcb_tx) {
3621
3622 txgbe_dcb_unpack_refill_cee(dcb_config,
3623 TXGBE_DCB_TX_CONFIG, refill);
3624 txgbe_dcb_unpack_max_cee(dcb_config, max);
3625 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3626 TXGBE_DCB_TX_CONFIG, bwgid);
3627 txgbe_dcb_unpack_tsa_cee(dcb_config,
3628 TXGBE_DCB_TX_CONFIG, tsa);
3629
3630 txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
3631 }
3632
3633
3634 txgbe_dcb_config_tc_stats_raptor(hw, dcb_config);
3635
3636
3637 if (dev->data->dev_conf.dcb_capability_en & RTE_ETH_DCB_PFC_SUPPORT) {
3638 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3639 for (i = 0; i < nb_tcs; i++) {
3640
3641
3642
3643
3644 hw->fc.high_water[i] = (pbsize * 3) / 4;
3645 hw->fc.low_water[i] = pbsize / 4;
3646
3647 tc = &dcb_config->tc_config[i];
3648 tc->pfc = txgbe_dcb_pfc_enabled;
3649 }
3650 txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
3651 if (dcb_config->num_tcs.pfc_tcs == RTE_ETH_4_TCS)
3652 pfc_en &= 0x0F;
3653 ret = txgbe_dcb_config_pfc(hw, pfc_en, map);
3654 }
3655
3656 return ret;
3657}
3658
3659void txgbe_configure_pb(struct rte_eth_dev *dev)
3660{
3661 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3662 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3663
3664 int hdrm;
3665 int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs;
3666
3667
3668 hdrm = 256;
3669
3670 hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
3671}
3672
3673void txgbe_configure_port(struct rte_eth_dev *dev)
3674{
3675 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3676 int i = 0;
3677 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ,
3678 0x9100, 0x9200,
3679 0x0000, 0x0000,
3680 0x0000, 0x0000};
3681
3682 PMD_INIT_FUNC_TRACE();
3683
3684
3685 wr32(hw, TXGBE_EXTAG,
3686 TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) |
3687 TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ));
3688
3689
3690 wr32m(hw, TXGBE_VLANCTL,
3691 TXGBE_VLANCTL_TPID_MASK,
3692 TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN));
3693 wr32m(hw, TXGBE_DMATXCTRL,
3694 TXGBE_DMATXCTRL_TPID_MASK,
3695 TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN));
3696
3697
3698 for (i = 0; i < 8; i++) {
3699 wr32m(hw, TXGBE_TAGTPID(i / 2),
3700 (i % 2 ? TXGBE_TAGTPID_MSB_MASK
3701 : TXGBE_TAGTPID_LSB_MASK),
3702 (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i])
3703 : TXGBE_TAGTPID_LSB(tpids[i])));
3704 }
3705
3706
3707 wr32(hw, TXGBE_VXLANPORT, 4789);
3708}
3709
3710
3711
3712
3713
3714void txgbe_configure_dcb(struct rte_eth_dev *dev)
3715{
3716 struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev);
3717 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3718
3719 PMD_INIT_FUNC_TRACE();
3720
3721
3722 if (dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_VMDQ_DCB &&
3723 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB &&
3724 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB_RSS)
3725 return;
3726
3727 if (dev->data->nb_rx_queues > RTE_ETH_DCB_NUM_QUEUES)
3728 return;
3729
3730
3731 txgbe_dcb_hw_configure(dev, dcb_cfg);
3732}
3733
3734
3735
3736
3737static void
3738txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
3739{
3740 struct rte_eth_vmdq_rx_conf *cfg;
3741 struct txgbe_hw *hw;
3742 enum rte_eth_nb_pools num_pools;
3743 uint32_t mrqc, vt_ctl, vlanctrl;
3744 uint32_t vmolr = 0;
3745 int i;
3746
3747 PMD_INIT_FUNC_TRACE();
3748 hw = TXGBE_DEV_HW(dev);
3749 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
3750 num_pools = cfg->nb_queue_pools;
3751
3752 txgbe_rss_disable(dev);
3753
3754
3755 mrqc = TXGBE_PORTCTL_NUMVT_64;
3756 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3757
3758
3759 vt_ctl = TXGBE_POOLCTL_RPLEN;
3760 if (cfg->enable_default_pool)
3761 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3762 else
3763 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3764
3765 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3766
3767 for (i = 0; i < (int)num_pools; i++) {
3768 vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
3769 wr32(hw, TXGBE_POOLETHCTL(i), vmolr);
3770 }
3771
3772
3773 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3774 vlanctrl |= TXGBE_VLANCTL_VFE;
3775 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3776
3777
3778 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3779 wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX);
3780
3781
3782 wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX);
3783 if (num_pools == RTE_ETH_64_POOLS)
3784 wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX);
3785
3786
3787
3788
3789
3790 wr32(hw, TXGBE_ETHADDRIDX, 0);
3791 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3792 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3793
3794
3795 for (i = 0; i < cfg->nb_pool_maps; i++) {
3796
3797 wr32(hw, TXGBE_PSRVLANIDX, i);
3798 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3799 TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id)));
3800
3801
3802
3803
3804
3805 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
3806 wr32(hw, TXGBE_PSRVLANPLM(0),
3807 (cfg->pool_map[i].pools & UINT32_MAX));
3808 else
3809 wr32(hw, TXGBE_PSRVLANPLM(1),
3810 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
3811 }
3812
3813
3814 if (cfg->enable_loop_back) {
3815 wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA);
3816 for (i = 0; i < 64; i++)
3817 wr32m(hw, TXGBE_POOLETHCTL(i),
3818 TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB);
3819 }
3820
3821 txgbe_flush(hw);
3822}
3823
3824
3825
3826
3827
3828static void
3829txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw)
3830{
3831 uint32_t reg;
3832 uint32_t q;
3833
3834 PMD_INIT_FUNC_TRACE();
3835
3836 wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX);
3837 wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX);
3838
3839
3840 reg = rd32(hw, TXGBE_ARBTXCTL);
3841 reg |= TXGBE_ARBTXCTL_DIA;
3842 wr32(hw, TXGBE_ARBTXCTL, reg);
3843
3844 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK,
3845 TXGBE_PORTCTL_NUMVT_64);
3846
3847
3848 for (q = 0; q < 128; q++) {
3849 u32 val = 1 << (q % 32);
3850 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3851 }
3852
3853
3854 reg = rd32(hw, TXGBE_ARBTXCTL);
3855 reg &= ~TXGBE_ARBTXCTL_DIA;
3856 wr32(hw, TXGBE_ARBTXCTL, reg);
3857
3858 txgbe_flush(hw);
3859}
3860
3861static int __rte_cold
3862txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq)
3863{
3864 struct txgbe_rx_entry *rxe = rxq->sw_ring;
3865 uint64_t dma_addr;
3866 unsigned int i;
3867
3868
3869 for (i = 0; i < rxq->nb_rx_desc; i++) {
3870 volatile struct txgbe_rx_desc *rxd;
3871 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
3872
3873 if (mbuf == NULL) {
3874 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
3875 (unsigned int)rxq->queue_id);
3876 return -ENOMEM;
3877 }
3878
3879 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
3880 mbuf->port = rxq->port_id;
3881
3882 dma_addr =
3883 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
3884 rxd = &rxq->rx_ring[i];
3885 TXGBE_RXD_HDRADDR(rxd, 0);
3886 TXGBE_RXD_PKTADDR(rxd, dma_addr);
3887 rxe[i].mbuf = mbuf;
3888 }
3889
3890 return 0;
3891}
3892
3893static int
3894txgbe_config_vf_rss(struct rte_eth_dev *dev)
3895{
3896 struct txgbe_hw *hw;
3897 uint32_t mrqc;
3898
3899 txgbe_rss_configure(dev);
3900
3901 hw = TXGBE_DEV_HW(dev);
3902
3903
3904 mrqc = rd32(hw, TXGBE_PORTCTL);
3905 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3906 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3907 case RTE_ETH_64_POOLS:
3908 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3909 break;
3910
3911 case RTE_ETH_32_POOLS:
3912 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3913 break;
3914
3915 default:
3916 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
3917 return -EINVAL;
3918 }
3919
3920 wr32(hw, TXGBE_PORTCTL, mrqc);
3921
3922 return 0;
3923}
3924
3925static int
3926txgbe_config_vf_default(struct rte_eth_dev *dev)
3927{
3928 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3929 uint32_t mrqc;
3930
3931 mrqc = rd32(hw, TXGBE_PORTCTL);
3932 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3933 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3934 case RTE_ETH_64_POOLS:
3935 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3936 break;
3937
3938 case RTE_ETH_32_POOLS:
3939 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3940 break;
3941
3942 case RTE_ETH_16_POOLS:
3943 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3944 break;
3945 default:
3946 PMD_INIT_LOG(ERR,
3947 "invalid pool number in IOV mode");
3948 return 0;
3949 }
3950
3951 wr32(hw, TXGBE_PORTCTL, mrqc);
3952
3953 return 0;
3954}
3955
3956static int
3957txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
3958{
3959 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3960
3961
3962
3963
3964 switch (dev->data->dev_conf.rxmode.mq_mode) {
3965 case RTE_ETH_MQ_RX_RSS:
3966 case RTE_ETH_MQ_RX_DCB_RSS:
3967 case RTE_ETH_MQ_RX_VMDQ_RSS:
3968 txgbe_rss_configure(dev);
3969 break;
3970
3971 case RTE_ETH_MQ_RX_VMDQ_DCB:
3972 txgbe_vmdq_dcb_configure(dev);
3973 break;
3974
3975 case RTE_ETH_MQ_RX_VMDQ_ONLY:
3976 txgbe_vmdq_rx_hw_configure(dev);
3977 break;
3978
3979 case RTE_ETH_MQ_RX_NONE:
3980 default:
3981
3982 txgbe_rss_disable(dev);
3983 break;
3984 }
3985 } else {
3986
3987
3988
3989 switch (dev->data->dev_conf.rxmode.mq_mode) {
3990 case RTE_ETH_MQ_RX_RSS:
3991 case RTE_ETH_MQ_RX_VMDQ_RSS:
3992 txgbe_config_vf_rss(dev);
3993 break;
3994 case RTE_ETH_MQ_RX_VMDQ_DCB:
3995 case RTE_ETH_MQ_RX_DCB:
3996
3997 txgbe_vmdq_dcb_configure(dev);
3998 break;
3999
4000 case RTE_ETH_MQ_RX_VMDQ_DCB_RSS:
4001 case RTE_ETH_MQ_RX_DCB_RSS:
4002 PMD_INIT_LOG(ERR,
4003 "Could not support DCB/RSS with VMDq & SRIOV");
4004 return -1;
4005 default:
4006 txgbe_config_vf_default(dev);
4007 break;
4008 }
4009 }
4010
4011 return 0;
4012}
4013
4014static int
4015txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
4016{
4017 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4018 uint32_t mtqc;
4019 uint32_t rttdcs;
4020
4021
4022 rttdcs = rd32(hw, TXGBE_ARBTXCTL);
4023 rttdcs |= TXGBE_ARBTXCTL_DIA;
4024 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
4025
4026 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4027
4028
4029
4030
4031 if (dev->data->dev_conf.txmode.mq_mode == RTE_ETH_MQ_TX_VMDQ_ONLY)
4032 txgbe_vmdq_tx_hw_configure(hw);
4033 else
4034 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0);
4035 } else {
4036 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4037
4038
4039
4040
4041 case RTE_ETH_64_POOLS:
4042 mtqc = TXGBE_PORTCTL_NUMVT_64;
4043 break;
4044 case RTE_ETH_32_POOLS:
4045 mtqc = TXGBE_PORTCTL_NUMVT_32;
4046 break;
4047 case RTE_ETH_16_POOLS:
4048 mtqc = TXGBE_PORTCTL_NUMVT_16;
4049 break;
4050 default:
4051 mtqc = 0;
4052 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
4053 }
4054 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc);
4055 }
4056
4057
4058 rttdcs &= ~TXGBE_ARBTXCTL_DIA;
4059 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
4060
4061 return 0;
4062}
4063
4064
4065
4066
4067
4068
4069static inline uint32_t
4070txgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
4071{
4072 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
4073
4074 uint16_t maxdesc =
4075 RTE_IPV4_MAX_PKT_LEN /
4076 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4077
4078 if (maxdesc >= 16)
4079 return TXGBE_RXCFG_RSCMAX_16;
4080 else if (maxdesc >= 8)
4081 return TXGBE_RXCFG_RSCMAX_8;
4082 else if (maxdesc >= 4)
4083 return TXGBE_RXCFG_RSCMAX_4;
4084 else
4085 return TXGBE_RXCFG_RSCMAX_1;
4086}
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097static int
4098txgbe_set_rsc(struct rte_eth_dev *dev)
4099{
4100 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4101 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4102 struct rte_eth_dev_info dev_info = { 0 };
4103 bool rsc_capable = false;
4104 uint16_t i;
4105 uint32_t rdrxctl;
4106 uint32_t rfctl;
4107
4108
4109 dev->dev_ops->dev_infos_get(dev, &dev_info);
4110 if (dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_TCP_LRO)
4111 rsc_capable = true;
4112
4113 if (!rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) {
4114 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4115 "support it");
4116 return -EINVAL;
4117 }
4118
4119
4120
4121 if ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) &&
4122 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) {
4123 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4124 "is disabled");
4125 return -EINVAL;
4126 }
4127
4128 rfctl = rd32(hw, TXGBE_PSRCTL);
4129 if (rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO))
4130 rfctl &= ~TXGBE_PSRCTL_RSCDIA;
4131 else
4132 rfctl |= TXGBE_PSRCTL_RSCDIA;
4133 wr32(hw, TXGBE_PSRCTL, rfctl);
4134
4135
4136 if (!(rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO))
4137 return 0;
4138
4139
4140 rdrxctl = rd32(hw, TXGBE_PSRCTL);
4141 rdrxctl |= TXGBE_PSRCTL_RSCACK;
4142 wr32(hw, TXGBE_PSRCTL, rdrxctl);
4143
4144
4145 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4146 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4147 uint32_t srrctl =
4148 rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4149 uint32_t psrtype =
4150 rd32(hw, TXGBE_POOLRSS(rxq->reg_idx));
4151 uint32_t eitr =
4152 rd32(hw, TXGBE_ITR(rxq->reg_idx));
4153
4154
4155
4156
4157 srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK;
4158 srrctl |= TXGBE_RXCFG_HDRLEN(128);
4159
4160
4161
4162
4163
4164
4165
4166 srrctl |= TXGBE_RXCFG_RSCENA;
4167 srrctl &= ~TXGBE_RXCFG_RSCMAX_MASK;
4168 srrctl |= txgbe_get_rscctl_maxdesc(rxq->mb_pool);
4169 psrtype |= TXGBE_POOLRSS_L4HDR;
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184 eitr &= ~TXGBE_ITR_IVAL_MASK;
4185 eitr |= TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
4186 eitr |= TXGBE_ITR_WRDSA;
4187
4188 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4189 wr32(hw, TXGBE_POOLRSS(rxq->reg_idx), psrtype);
4190 wr32(hw, TXGBE_ITR(rxq->reg_idx), eitr);
4191
4192
4193
4194
4195
4196 txgbe_set_ivar_map(hw, 0, rxq->reg_idx, i);
4197 }
4198
4199 dev->data->lro = 1;
4200
4201 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
4202
4203 return 0;
4204}
4205
4206void __rte_cold
4207txgbe_set_rx_function(struct rte_eth_dev *dev)
4208{
4209 uint16_t i;
4210 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4211
4212
4213
4214
4215
4216
4217
4218
4219 if (dev->data->lro) {
4220 if (adapter->rx_bulk_alloc_allowed) {
4221 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4222 "allocation version");
4223 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4224 } else {
4225 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4226 "allocation version");
4227 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4228 }
4229 } else if (dev->data->scattered_rx) {
4230
4231
4232
4233
4234 if (adapter->rx_bulk_alloc_allowed) {
4235 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4236 "allocation callback (port=%d).",
4237 dev->data->port_id);
4238 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4239 } else {
4240 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
4241 "single allocation) "
4242 "Scattered Rx callback "
4243 "(port=%d).",
4244 dev->data->port_id);
4245
4246 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4247 }
4248
4249
4250
4251
4252
4253
4254
4255 } else if (adapter->rx_bulk_alloc_allowed) {
4256 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4257 "satisfied. Rx Burst Bulk Alloc function "
4258 "will be used on port=%d.",
4259 dev->data->port_id);
4260
4261 dev->rx_pkt_burst = txgbe_recv_pkts_bulk_alloc;
4262 } else {
4263 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4264 "satisfied, or Scattered Rx is requested "
4265 "(port=%d).",
4266 dev->data->port_id);
4267
4268 dev->rx_pkt_burst = txgbe_recv_pkts;
4269 }
4270
4271#ifdef RTE_LIB_SECURITY
4272 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4273 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4274
4275 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4276 RTE_ETH_RX_OFFLOAD_SECURITY);
4277 }
4278#endif
4279}
4280
4281
4282
4283
4284int __rte_cold
4285txgbe_dev_rx_init(struct rte_eth_dev *dev)
4286{
4287 struct txgbe_hw *hw;
4288 struct txgbe_rx_queue *rxq;
4289 uint64_t bus_addr;
4290 uint32_t fctrl;
4291 uint32_t hlreg0;
4292 uint32_t srrctl;
4293 uint32_t rdrxctl;
4294 uint32_t rxcsum;
4295 uint16_t buf_size;
4296 uint16_t i;
4297 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4298 int rc;
4299
4300 PMD_INIT_FUNC_TRACE();
4301 hw = TXGBE_DEV_HW(dev);
4302
4303
4304
4305
4306
4307 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, 0);
4308 wr32m(hw, TXGBE_PBRXCTL, TXGBE_PBRXCTL_ENA, 0);
4309
4310
4311 fctrl = rd32(hw, TXGBE_PSRCTL);
4312 fctrl |= TXGBE_PSRCTL_BCA;
4313 wr32(hw, TXGBE_PSRCTL, fctrl);
4314
4315
4316
4317
4318 hlreg0 = rd32(hw, TXGBE_SECRXCTL);
4319 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
4320 hlreg0 &= ~TXGBE_SECRXCTL_CRCSTRIP;
4321 else
4322 hlreg0 |= TXGBE_SECRXCTL_CRCSTRIP;
4323 wr32(hw, TXGBE_SECRXCTL, hlreg0);
4324
4325
4326
4327
4328 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4329 TXGBE_FRMSZ_MAX(dev->data->mtu + TXGBE_ETH_OVERHEAD));
4330
4331
4332
4333
4334 hlreg0 = rd32(hw, TXGBE_PSRCTL);
4335 if (hw->mac.type == txgbe_mac_raptor &&
4336 dev->data->dev_conf.lpbk_mode)
4337 hlreg0 |= TXGBE_PSRCTL_LBENA;
4338 else
4339 hlreg0 &= ~TXGBE_PSRCTL_LBENA;
4340
4341 wr32(hw, TXGBE_PSRCTL, hlreg0);
4342
4343
4344
4345
4346
4347 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
4348
4349
4350 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4351 rxq = dev->data->rx_queues[i];
4352
4353
4354
4355
4356
4357 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
4358 rxq->crc_len = RTE_ETHER_CRC_LEN;
4359 else
4360 rxq->crc_len = 0;
4361
4362
4363 bus_addr = rxq->rx_ring_phys_addr;
4364 wr32(hw, TXGBE_RXBAL(rxq->reg_idx),
4365 (uint32_t)(bus_addr & BIT_MASK32));
4366 wr32(hw, TXGBE_RXBAH(rxq->reg_idx),
4367 (uint32_t)(bus_addr >> 32));
4368 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4369 wr32(hw, TXGBE_RXWP(rxq->reg_idx), 0);
4370
4371 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
4372
4373
4374 if (rxq->drop_en)
4375 srrctl |= TXGBE_RXCFG_DROP;
4376
4377
4378
4379
4380
4381
4382
4383 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4384 RTE_PKTMBUF_HEADROOM);
4385 buf_size = ROUND_UP(buf_size, 0x1 << 10);
4386 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size);
4387
4388 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4389
4390
4391 if (dev->data->mtu + TXGBE_ETH_OVERHEAD +
4392 2 * RTE_VLAN_HLEN > buf_size)
4393 dev->data->scattered_rx = 1;
4394 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
4395 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
4396 }
4397
4398 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
4399 dev->data->scattered_rx = 1;
4400
4401
4402
4403
4404 txgbe_dev_mq_rx_configure(dev);
4405
4406
4407
4408
4409
4410
4411 rxcsum = rd32(hw, TXGBE_PSRCTL);
4412 rxcsum |= TXGBE_PSRCTL_PCSD;
4413 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM)
4414 rxcsum |= TXGBE_PSRCTL_L4CSUM;
4415 else
4416 rxcsum &= ~TXGBE_PSRCTL_L4CSUM;
4417
4418 wr32(hw, TXGBE_PSRCTL, rxcsum);
4419
4420 if (hw->mac.type == txgbe_mac_raptor) {
4421 rdrxctl = rd32(hw, TXGBE_SECRXCTL);
4422 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
4423 rdrxctl &= ~TXGBE_SECRXCTL_CRCSTRIP;
4424 else
4425 rdrxctl |= TXGBE_SECRXCTL_CRCSTRIP;
4426 wr32(hw, TXGBE_SECRXCTL, rdrxctl);
4427 }
4428
4429 rc = txgbe_set_rsc(dev);
4430 if (rc)
4431 return rc;
4432
4433 txgbe_set_rx_function(dev);
4434
4435 return 0;
4436}
4437
4438
4439
4440
4441void __rte_cold
4442txgbe_dev_tx_init(struct rte_eth_dev *dev)
4443{
4444 struct txgbe_hw *hw;
4445 struct txgbe_tx_queue *txq;
4446 uint64_t bus_addr;
4447 uint16_t i;
4448
4449 PMD_INIT_FUNC_TRACE();
4450 hw = TXGBE_DEV_HW(dev);
4451
4452
4453 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4454 txq = dev->data->tx_queues[i];
4455
4456 bus_addr = txq->tx_ring_phys_addr;
4457 wr32(hw, TXGBE_TXBAL(txq->reg_idx),
4458 (uint32_t)(bus_addr & BIT_MASK32));
4459 wr32(hw, TXGBE_TXBAH(txq->reg_idx),
4460 (uint32_t)(bus_addr >> 32));
4461 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_BUFLEN_MASK,
4462 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
4463
4464 wr32(hw, TXGBE_TXRP(txq->reg_idx), 0);
4465 wr32(hw, TXGBE_TXWP(txq->reg_idx), 0);
4466 }
4467
4468
4469 txgbe_dev_mq_tx_configure(dev);
4470}
4471
4472
4473
4474
4475static inline void __rte_cold
4476txgbe_setup_loopback_link_raptor(struct txgbe_hw *hw)
4477{
4478 PMD_INIT_FUNC_TRACE();
4479
4480 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_LB, TXGBE_MACRXCFG_LB);
4481
4482 msec_delay(50);
4483}
4484
4485
4486
4487
4488int __rte_cold
4489txgbe_dev_rxtx_start(struct rte_eth_dev *dev)
4490{
4491 struct txgbe_hw *hw;
4492 struct txgbe_tx_queue *txq;
4493 struct txgbe_rx_queue *rxq;
4494 uint32_t dmatxctl;
4495 uint32_t rxctrl;
4496 uint16_t i;
4497 int ret = 0;
4498
4499 PMD_INIT_FUNC_TRACE();
4500 hw = TXGBE_DEV_HW(dev);
4501
4502 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4503 txq = dev->data->tx_queues[i];
4504
4505 wr32m(hw, TXGBE_TXCFG(txq->reg_idx),
4506 TXGBE_TXCFG_HTHRESH_MASK |
4507 TXGBE_TXCFG_WTHRESH_MASK,
4508 TXGBE_TXCFG_HTHRESH(txq->hthresh) |
4509 TXGBE_TXCFG_WTHRESH(txq->wthresh));
4510 }
4511
4512 dmatxctl = rd32(hw, TXGBE_DMATXCTRL);
4513 dmatxctl |= TXGBE_DMATXCTRL_ENA;
4514 wr32(hw, TXGBE_DMATXCTRL, dmatxctl);
4515
4516 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4517 txq = dev->data->tx_queues[i];
4518 if (!txq->tx_deferred_start) {
4519 ret = txgbe_dev_tx_queue_start(dev, i);
4520 if (ret < 0)
4521 return ret;
4522 }
4523 }
4524
4525 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4526 rxq = dev->data->rx_queues[i];
4527 if (!rxq->rx_deferred_start) {
4528 ret = txgbe_dev_rx_queue_start(dev, i);
4529 if (ret < 0)
4530 return ret;
4531 }
4532 }
4533
4534
4535 rxctrl = rd32(hw, TXGBE_PBRXCTL);
4536 rxctrl |= TXGBE_PBRXCTL_ENA;
4537 hw->mac.enable_rx_dma(hw, rxctrl);
4538
4539
4540 if (hw->mac.type == txgbe_mac_raptor &&
4541 dev->data->dev_conf.lpbk_mode)
4542 txgbe_setup_loopback_link_raptor(hw);
4543
4544#ifdef RTE_LIB_SECURITY
4545 if ((dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SECURITY) ||
4546 (dev->data->dev_conf.txmode.offloads & RTE_ETH_TX_OFFLOAD_SECURITY)) {
4547 ret = txgbe_crypto_enable_ipsec(dev);
4548 if (ret != 0) {
4549 PMD_DRV_LOG(ERR,
4550 "txgbe_crypto_enable_ipsec fails with %d.",
4551 ret);
4552 return ret;
4553 }
4554 }
4555#endif
4556
4557 return 0;
4558}
4559
4560void
4561txgbe_dev_save_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4562{
4563 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4564 *(reg++) = rd32(hw, TXGBE_RXBAL(rx_queue_id));
4565 *(reg++) = rd32(hw, TXGBE_RXBAH(rx_queue_id));
4566 *(reg++) = rd32(hw, TXGBE_RXCFG(rx_queue_id));
4567}
4568
4569void
4570txgbe_dev_store_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4571{
4572 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4573 wr32(hw, TXGBE_RXBAL(rx_queue_id), *(reg++));
4574 wr32(hw, TXGBE_RXBAH(rx_queue_id), *(reg++));
4575 wr32(hw, TXGBE_RXCFG(rx_queue_id), *(reg++) & ~TXGBE_RXCFG_ENA);
4576}
4577
4578void
4579txgbe_dev_save_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4580{
4581 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4582 *(reg++) = rd32(hw, TXGBE_TXBAL(tx_queue_id));
4583 *(reg++) = rd32(hw, TXGBE_TXBAH(tx_queue_id));
4584 *(reg++) = rd32(hw, TXGBE_TXCFG(tx_queue_id));
4585}
4586
4587void
4588txgbe_dev_store_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4589{
4590 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4591 wr32(hw, TXGBE_TXBAL(tx_queue_id), *(reg++));
4592 wr32(hw, TXGBE_TXBAH(tx_queue_id), *(reg++));
4593 wr32(hw, TXGBE_TXCFG(tx_queue_id), *(reg++) & ~TXGBE_TXCFG_ENA);
4594}
4595
4596
4597
4598
4599int __rte_cold
4600txgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4601{
4602 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4603 struct txgbe_rx_queue *rxq;
4604 uint32_t rxdctl;
4605 int poll_ms;
4606
4607 PMD_INIT_FUNC_TRACE();
4608
4609 rxq = dev->data->rx_queues[rx_queue_id];
4610
4611
4612 if (txgbe_alloc_rx_queue_mbufs(rxq) != 0) {
4613 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
4614 rx_queue_id);
4615 return -1;
4616 }
4617 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4618 rxdctl |= TXGBE_RXCFG_ENA;
4619 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxdctl);
4620
4621
4622 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4623 do {
4624 rte_delay_ms(1);
4625 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4626 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA));
4627 if (!poll_ms)
4628 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
4629 rte_wmb();
4630 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4631 wr32(hw, TXGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
4632 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4633
4634 return 0;
4635}
4636
4637
4638
4639
4640int __rte_cold
4641txgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4642{
4643 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4644 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4645 struct txgbe_rx_queue *rxq;
4646 uint32_t rxdctl;
4647 int poll_ms;
4648
4649 PMD_INIT_FUNC_TRACE();
4650
4651 rxq = dev->data->rx_queues[rx_queue_id];
4652
4653 txgbe_dev_save_rx_queue(hw, rxq->reg_idx);
4654 wr32m(hw, TXGBE_RXCFG(rxq->reg_idx), TXGBE_RXCFG_ENA, 0);
4655
4656
4657 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4658 do {
4659 rte_delay_ms(1);
4660 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4661 } while (--poll_ms && (rxdctl & TXGBE_RXCFG_ENA));
4662 if (!poll_ms)
4663 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
4664
4665 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4666 txgbe_dev_store_rx_queue(hw, rxq->reg_idx);
4667
4668 txgbe_rx_queue_release_mbufs(rxq);
4669 txgbe_reset_rx_queue(adapter, rxq);
4670 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4671
4672 return 0;
4673}
4674
4675
4676
4677
4678int __rte_cold
4679txgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4680{
4681 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4682 struct txgbe_tx_queue *txq;
4683 uint32_t txdctl;
4684 int poll_ms;
4685
4686 PMD_INIT_FUNC_TRACE();
4687
4688 txq = dev->data->tx_queues[tx_queue_id];
4689 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA);
4690
4691
4692 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4693 do {
4694 rte_delay_ms(1);
4695 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4696 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA));
4697 if (!poll_ms)
4698 PMD_INIT_LOG(ERR, "Could not enable "
4699 "Tx Queue %d", tx_queue_id);
4700
4701 rte_wmb();
4702 wr32(hw, TXGBE_TXWP(txq->reg_idx), txq->tx_tail);
4703 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4704
4705 return 0;
4706}
4707
4708
4709
4710
4711int __rte_cold
4712txgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4713{
4714 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4715 struct txgbe_tx_queue *txq;
4716 uint32_t txdctl;
4717 uint32_t txtdh, txtdt;
4718 int poll_ms;
4719
4720 PMD_INIT_FUNC_TRACE();
4721
4722 txq = dev->data->tx_queues[tx_queue_id];
4723
4724
4725 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4726 do {
4727 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4728 txtdh = rd32(hw, TXGBE_TXRP(txq->reg_idx));
4729 txtdt = rd32(hw, TXGBE_TXWP(txq->reg_idx));
4730 } while (--poll_ms && (txtdh != txtdt));
4731 if (!poll_ms)
4732 PMD_INIT_LOG(ERR,
4733 "Tx Queue %d is not empty when stopping.",
4734 tx_queue_id);
4735
4736 txgbe_dev_save_tx_queue(hw, txq->reg_idx);
4737 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, 0);
4738
4739
4740 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4741 do {
4742 rte_delay_ms(1);
4743 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4744 } while (--poll_ms && (txdctl & TXGBE_TXCFG_ENA));
4745 if (!poll_ms)
4746 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
4747 tx_queue_id);
4748
4749 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4750 txgbe_dev_store_tx_queue(hw, txq->reg_idx);
4751
4752 if (txq->ops != NULL) {
4753 txq->ops->release_mbufs(txq);
4754 txq->ops->reset(txq);
4755 }
4756 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4757
4758 return 0;
4759}
4760
4761void
4762txgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4763 struct rte_eth_rxq_info *qinfo)
4764{
4765 struct txgbe_rx_queue *rxq;
4766
4767 rxq = dev->data->rx_queues[queue_id];
4768
4769 qinfo->mp = rxq->mb_pool;
4770 qinfo->scattered_rx = dev->data->scattered_rx;
4771 qinfo->nb_desc = rxq->nb_rx_desc;
4772
4773 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
4774 qinfo->conf.rx_drop_en = rxq->drop_en;
4775 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
4776 qinfo->conf.offloads = rxq->offloads;
4777}
4778
4779void
4780txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4781 struct rte_eth_txq_info *qinfo)
4782{
4783 struct txgbe_tx_queue *txq;
4784
4785 txq = dev->data->tx_queues[queue_id];
4786
4787 qinfo->nb_desc = txq->nb_tx_desc;
4788
4789 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
4790 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
4791 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
4792
4793 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
4794 qinfo->conf.offloads = txq->offloads;
4795 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
4796}
4797
4798
4799
4800
4801int __rte_cold
4802txgbevf_dev_rx_init(struct rte_eth_dev *dev)
4803{
4804 struct txgbe_hw *hw;
4805 struct txgbe_rx_queue *rxq;
4806 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
4807 uint64_t bus_addr;
4808 uint32_t srrctl, psrtype;
4809 uint16_t buf_size;
4810 uint16_t i;
4811 int ret;
4812
4813 PMD_INIT_FUNC_TRACE();
4814 hw = TXGBE_DEV_HW(dev);
4815
4816 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
4817 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
4818 "it should be power of 2");
4819 return -1;
4820 }
4821
4822 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
4823 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
4824 "it should be equal to or less than %d",
4825 hw->mac.max_rx_queues);
4826 return -1;
4827 }
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843 if (txgbevf_rlpml_set_vf(hw,
4844 (uint16_t)dev->data->mtu + TXGBE_ETH_OVERHEAD)) {
4845 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.",
4846 dev->data->mtu + TXGBE_ETH_OVERHEAD);
4847 return -EINVAL;
4848 }
4849
4850
4851
4852
4853
4854 rxmode->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
4855
4856
4857 psrtype = TXGBE_VFPLCFG_PSRL4HDR |
4858 TXGBE_VFPLCFG_PSRL4HDR |
4859 TXGBE_VFPLCFG_PSRL2HDR |
4860 TXGBE_VFPLCFG_PSRTUNHDR |
4861 TXGBE_VFPLCFG_PSRTUNMAC;
4862 wr32(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_PSR(psrtype));
4863
4864
4865 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4866 rxq = dev->data->rx_queues[i];
4867
4868
4869 ret = txgbe_alloc_rx_queue_mbufs(rxq);
4870 if (ret)
4871 return ret;
4872
4873
4874 bus_addr = rxq->rx_ring_phys_addr;
4875
4876 wr32(hw, TXGBE_RXBAL(i),
4877 (uint32_t)(bus_addr & BIT_MASK32));
4878 wr32(hw, TXGBE_RXBAH(i),
4879 (uint32_t)(bus_addr >> 32));
4880 wr32(hw, TXGBE_RXRP(i), 0);
4881 wr32(hw, TXGBE_RXWP(i), 0);
4882
4883
4884 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
4885
4886
4887 if (rxq->drop_en)
4888 srrctl |= TXGBE_RXCFG_DROP;
4889
4890
4891
4892
4893
4894
4895
4896 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4897 RTE_PKTMBUF_HEADROOM);
4898 buf_size = ROUND_UP(buf_size, 1 << 10);
4899 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size);
4900
4901
4902
4903
4904 wr32(hw, TXGBE_RXCFG(i), srrctl);
4905
4906 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER ||
4907
4908 (dev->data->mtu + TXGBE_ETH_OVERHEAD +
4909 2 * RTE_VLAN_HLEN) > buf_size) {
4910 if (!dev->data->scattered_rx)
4911 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
4912 dev->data->scattered_rx = 1;
4913 }
4914
4915 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
4916 rxmode->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
4917 }
4918
4919
4920
4921
4922 txgbe_dev_mq_rx_configure(dev);
4923
4924 txgbe_set_rx_function(dev);
4925
4926 return 0;
4927}
4928
4929
4930
4931
4932void __rte_cold
4933txgbevf_dev_tx_init(struct rte_eth_dev *dev)
4934{
4935 struct txgbe_hw *hw;
4936 struct txgbe_tx_queue *txq;
4937 uint64_t bus_addr;
4938 uint16_t i;
4939
4940 PMD_INIT_FUNC_TRACE();
4941 hw = TXGBE_DEV_HW(dev);
4942
4943
4944 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4945 txq = dev->data->tx_queues[i];
4946 bus_addr = txq->tx_ring_phys_addr;
4947 wr32(hw, TXGBE_TXBAL(i),
4948 (uint32_t)(bus_addr & BIT_MASK32));
4949 wr32(hw, TXGBE_TXBAH(i),
4950 (uint32_t)(bus_addr >> 32));
4951 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_BUFLEN_MASK,
4952 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
4953
4954 wr32(hw, TXGBE_TXRP(i), 0);
4955 wr32(hw, TXGBE_TXWP(i), 0);
4956 }
4957}
4958
4959
4960
4961
4962void __rte_cold
4963txgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
4964{
4965 struct txgbe_hw *hw;
4966 struct txgbe_tx_queue *txq;
4967 struct txgbe_rx_queue *rxq;
4968 uint32_t txdctl;
4969 uint32_t rxdctl;
4970 uint16_t i;
4971 int poll_ms;
4972
4973 PMD_INIT_FUNC_TRACE();
4974 hw = TXGBE_DEV_HW(dev);
4975
4976 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4977 txq = dev->data->tx_queues[i];
4978
4979 wr32m(hw, TXGBE_TXCFG(txq->reg_idx),
4980 TXGBE_TXCFG_HTHRESH_MASK |
4981 TXGBE_TXCFG_WTHRESH_MASK,
4982 TXGBE_TXCFG_HTHRESH(txq->hthresh) |
4983 TXGBE_TXCFG_WTHRESH(txq->wthresh));
4984 }
4985
4986 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4987 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA);
4988
4989 poll_ms = 10;
4990
4991 do {
4992 rte_delay_ms(1);
4993 txdctl = rd32(hw, TXGBE_TXCFG(i));
4994 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA));
4995 if (!poll_ms)
4996 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
4997 }
4998 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4999 rxq = dev->data->rx_queues[i];
5000
5001 wr32m(hw, TXGBE_RXCFG(i), TXGBE_RXCFG_ENA, TXGBE_RXCFG_ENA);
5002
5003
5004 poll_ms = 10;
5005 do {
5006 rte_delay_ms(1);
5007 rxdctl = rd32(hw, TXGBE_RXCFG(i));
5008 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA));
5009 if (!poll_ms)
5010 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
5011 rte_wmb();
5012 wr32(hw, TXGBE_RXWP(i), rxq->nb_rx_desc - 1);
5013 }
5014}
5015
5016int
5017txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out,
5018 const struct rte_flow_action_rss *in)
5019{
5020 if (in->key_len > RTE_DIM(out->key) ||
5021 in->queue_num > RTE_DIM(out->queue))
5022 return -EINVAL;
5023 out->conf = (struct rte_flow_action_rss){
5024 .func = in->func,
5025 .level = in->level,
5026 .types = in->types,
5027 .key_len = in->key_len,
5028 .queue_num = in->queue_num,
5029 .key = memcpy(out->key, in->key, in->key_len),
5030 .queue = memcpy(out->queue, in->queue,
5031 sizeof(*in->queue) * in->queue_num),
5032 };
5033 return 0;
5034}
5035
5036int
5037txgbe_action_rss_same(const struct rte_flow_action_rss *comp,
5038 const struct rte_flow_action_rss *with)
5039{
5040 return (comp->func == with->func &&
5041 comp->level == with->level &&
5042 comp->types == with->types &&
5043 comp->key_len == with->key_len &&
5044 comp->queue_num == with->queue_num &&
5045 !memcmp(comp->key, with->key, with->key_len) &&
5046 !memcmp(comp->queue, with->queue,
5047 sizeof(*with->queue) * with->queue_num));
5048}
5049
5050int
5051txgbe_config_rss_filter(struct rte_eth_dev *dev,
5052 struct txgbe_rte_flow_rss_conf *conf, bool add)
5053{
5054 struct txgbe_hw *hw;
5055 uint32_t reta;
5056 uint16_t i;
5057 uint16_t j;
5058 struct rte_eth_rss_conf rss_conf = {
5059 .rss_key = conf->conf.key_len ?
5060 (void *)(uintptr_t)conf->conf.key : NULL,
5061 .rss_key_len = conf->conf.key_len,
5062 .rss_hf = conf->conf.types,
5063 };
5064 struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev);
5065
5066 PMD_INIT_FUNC_TRACE();
5067 hw = TXGBE_DEV_HW(dev);
5068
5069 if (!add) {
5070 if (txgbe_action_rss_same(&filter_info->rss_info.conf,
5071 &conf->conf)) {
5072 txgbe_rss_disable(dev);
5073 memset(&filter_info->rss_info, 0,
5074 sizeof(struct txgbe_rte_flow_rss_conf));
5075 return 0;
5076 }
5077 return -EINVAL;
5078 }
5079
5080 if (filter_info->rss_info.conf.queue_num)
5081 return -EINVAL;
5082
5083
5084
5085
5086 reta = 0;
5087 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) {
5088 if (j == conf->conf.queue_num)
5089 j = 0;
5090 reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF);
5091 if ((i & 3) == 3)
5092 wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
5093 }
5094
5095
5096
5097
5098 if ((rss_conf.rss_hf & TXGBE_RSS_OFFLOAD_ALL) == 0) {
5099 txgbe_rss_disable(dev);
5100 return 0;
5101 }
5102 if (rss_conf.rss_key == NULL)
5103 rss_conf.rss_key = rss_intel_key;
5104 txgbe_dev_rss_hash_update(dev, &rss_conf);
5105
5106 if (txgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))
5107 return -EINVAL;
5108
5109 return 0;
5110}
5111
5112