LXR dpdk/drivers/net/axgbe/axgbe_rxtx_vec

   1/*   SPDX-License-Identifier: BSD-3-Clause
   2 *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3 *   Copyright(c) 2018 Synopsys, Inc. All rights reserved.
   4 */
   5
   6#include "axgbe_ethdev.h"
   7#include "axgbe_rxtx.h"
   8#include "axgbe_phy.h"
   9
  10#include <rte_time.h>
  11#include <rte_mempool.h>
  12#include <rte_mbuf.h>
  13
  14/* Useful to avoid shifting for every descriptor preparation */
  15#define TX_DESC_CTRL_FLAGS 0xb000000000000000
  16#define TX_DESC_CTRL_FLAG_TMST 0x40000000
  17#define TX_FREE_BULK       8
  18#define TX_FREE_BULK_CHECK (TX_FREE_BULK - 1)
  19
  20static inline void
  21axgbe_vec_tx(volatile struct axgbe_tx_desc *desc,
  22             struct rte_mbuf *mbuf)
  23{
  24        uint64_t tmst_en = 0;
  25        /* Timestamp enablement check */
  26        if (mbuf->ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST)
  27                tmst_en = TX_DESC_CTRL_FLAG_TMST;
  28        __m128i descriptor = _mm_set_epi64x((uint64_t)mbuf->pkt_len << 32 |
  29                                            TX_DESC_CTRL_FLAGS | mbuf->data_len
  30                                            | tmst_en,
  31                                            mbuf->buf_iova
  32                                            + mbuf->data_off);
  33        _mm_store_si128((__m128i *)desc, descriptor);
  34}
  35
  36static void
  37axgbe_xmit_cleanup_vec(struct axgbe_tx_queue *txq)
  38{
  39        volatile struct axgbe_tx_desc *desc;
  40        int idx, i;
  41
  42        idx = AXGBE_GET_DESC_IDX(txq, txq->dirty + txq->free_batch_cnt
  43                                 - 1);
  44        desc = &txq->desc[idx];
  45        if (desc->desc3 & AXGBE_DESC_OWN)
  46                return;
  47        /* memset avoided for desc ctrl fields since in vec_tx path
  48         * all 128 bits are populated
  49         */
  50        for (i = 0; i < txq->free_batch_cnt; i++, idx--)
  51                rte_pktmbuf_free_seg(txq->sw_ring[idx]);
  52
  53
  54        txq->dirty += txq->free_batch_cnt;
  55        txq->nb_desc_free += txq->free_batch_cnt;
  56}
  57
  58uint16_t
  59axgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
  60                    uint16_t nb_pkts)
  61{
  62        PMD_INIT_FUNC_TRACE();
  63
  64        struct axgbe_tx_queue *txq;
  65        uint16_t idx, nb_commit, loop, i;
  66        uint32_t tail_addr;
  67
  68        txq  = (struct axgbe_tx_queue *)tx_queue;
  69        if (txq->nb_desc_free < txq->free_thresh) {
  70                axgbe_xmit_cleanup_vec(txq);
  71                if (unlikely(txq->nb_desc_free == 0))
  72                        return 0;
  73        }
  74        nb_pkts = RTE_MIN(txq->nb_desc_free, nb_pkts);
  75        nb_commit = nb_pkts;
  76        idx = AXGBE_GET_DESC_IDX(txq, txq->cur);
  77        loop = txq->nb_desc - idx;
  78        if (nb_commit >= loop) {
  79                for (i = 0; i < loop; ++i, ++idx, ++tx_pkts) {
  80                        axgbe_vec_tx(&txq->desc[idx], *tx_pkts);
  81                        txq->sw_ring[idx] = *tx_pkts;
  82                }
  83                nb_commit -= loop;
  84                idx = 0;
  85        }
  86        for (i = 0; i < nb_commit; ++i, ++idx, ++tx_pkts) {
  87                axgbe_vec_tx(&txq->desc[idx], *tx_pkts);
  88                txq->sw_ring[idx] = *tx_pkts;
  89        }
  90        txq->cur += nb_pkts;
  91        tail_addr = (uint32_t)(txq->ring_phys_addr +
  92                               idx * sizeof(struct axgbe_tx_desc));
  93        /* Update tail reg with next immediate address to kick Tx DMA channel*/
  94        rte_write32(tail_addr, (void *)txq->dma_tail_reg);
  95        txq->pkts += nb_pkts;
  96        txq->nb_desc_free -= nb_pkts;
  97
  98        return nb_pkts;
  99}
 100