LXR dpdk/drivers/net/sfc/sfc

   1/* SPDX-License-Identifier: BSD-3-Clause
   2 *
   3 * Copyright(c) 2019-2021 Xilinx, Inc.
   4 * Copyright(c) 2017-2019 Solarflare Communications Inc.
   5 *
   6 * This software was jointly developed between OKTET Labs (under contract
   7 * for Solarflare) and Solarflare Communications, Inc.
   8 */
   9
  10#ifndef _SFC_EF10_H
  11#define _SFC_EF10_H
  12
  13#include "sfc_debug.h"
  14
  15#ifdef __cplusplus
  16extern "C" {
  17#endif
  18
  19/* Number of events in one cache line */
  20#define SFC_EF10_EV_PER_CACHE_LINE \
  21        (RTE_CACHE_LINE_SIZE / sizeof(efx_qword_t))
  22
  23#define SFC_EF10_EV_QCLEAR_MASK         (~(SFC_EF10_EV_PER_CACHE_LINE - 1))
  24
  25/*
  26 * Use simple libefx-based implementation of the
  27 * sfc_ef10_ev_qclear_cache_line() if SSE2 is not available
  28 * since optimized implementation uses __m128i intrinsics.
  29 */
  30#ifndef __SSE2__
  31#define SFC_EF10_EV_QCLEAR_USE_EFX
  32#endif
  33
  34#if defined(SFC_EF10_EV_QCLEAR_USE_EFX)
  35static inline void
  36sfc_ef10_ev_qclear_cache_line(void *ptr)
  37{
  38        efx_qword_t *entry = ptr;
  39        unsigned int i;
  40
  41        for (i = 0; i < SFC_EF10_EV_PER_CACHE_LINE; ++i)
  42                EFX_SET_QWORD(entry[i]);
  43}
  44#else
  45/*
  46 * It is possible to do it using AVX2 and AVX512F, but it shows less
  47 * performance.
  48 */
  49static inline void
  50sfc_ef10_ev_qclear_cache_line(void *ptr)
  51{
  52        const efsys_uint128_t val = _mm_set1_epi64x(UINT64_MAX);
  53        efsys_uint128_t *addr = ptr;
  54        unsigned int i;
  55
  56        RTE_BUILD_BUG_ON(sizeof(val) > RTE_CACHE_LINE_SIZE);
  57        RTE_BUILD_BUG_ON(RTE_CACHE_LINE_SIZE % sizeof(val) != 0);
  58
  59        for (i = 0; i < RTE_CACHE_LINE_SIZE / sizeof(val); ++i)
  60                _mm_store_si128(&addr[i], val);
  61}
  62#endif
  63
  64static inline void
  65sfc_ef10_ev_qclear(efx_qword_t *hw_ring, unsigned int ptr_mask,
  66                   unsigned int old_read_ptr, unsigned int read_ptr)
  67{
  68        const unsigned int clear_ptr = read_ptr & SFC_EF10_EV_QCLEAR_MASK;
  69        unsigned int old_clear_ptr = old_read_ptr & SFC_EF10_EV_QCLEAR_MASK;
  70
  71        while (old_clear_ptr != clear_ptr) {
  72                sfc_ef10_ev_qclear_cache_line(
  73                        &hw_ring[old_clear_ptr & ptr_mask]);
  74                old_clear_ptr += SFC_EF10_EV_PER_CACHE_LINE;
  75        }
  76
  77        /*
  78         * No barriers here.
  79         * Functions which push doorbell should care about correct
  80         * ordering: store instructions which fill in EvQ ring should be
  81         * retired from CPU and DMA sync before doorbell which will allow
  82         * to use these event entries.
  83         */
  84}
  85
  86static inline bool
  87sfc_ef10_ev_present(const efx_qword_t ev)
  88{
  89        return ~EFX_QWORD_FIELD(ev, EFX_DWORD_0) |
  90               ~EFX_QWORD_FIELD(ev, EFX_DWORD_1);
  91}
  92
  93
  94/**
  95 * Alignment requirement for value written to RX WPTR:
  96 * the WPTR must be aligned to an 8 descriptor boundary.
  97 */
  98#define SFC_EF10_RX_WPTR_ALIGN  8u
  99
 100static inline void
 101sfc_ef10_rx_qpush(volatile void *doorbell, unsigned int added,
 102                  unsigned int ptr_mask)
 103{
 104        efx_dword_t dword;
 105
 106        /* Hardware has alignment restriction for WPTR */
 107        RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0);
 108        SFC_ASSERT(RTE_ALIGN(added, SFC_EF10_RX_WPTR_ALIGN) == added);
 109
 110        EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, added & ptr_mask);
 111
 112        /* DMA sync to device is not required */
 113
 114        /*
 115         * rte_write32() has rte_io_wmb() which guarantees that the STORE
 116         * operations (i.e. Rx and event descriptor updates) that precede
 117         * the rte_io_wmb() call are visible to NIC before the STORE
 118         * operations that follow it (i.e. doorbell write).
 119         */
 120        rte_write32(dword.ed_u32[0], doorbell);
 121}
 122
 123static inline void
 124sfc_ef10_ev_qprime(volatile void *qprime, unsigned int read_ptr,
 125                  unsigned int ptr_mask)
 126{
 127        efx_dword_t dword;
 128
 129        EFX_POPULATE_DWORD_1(dword, ERF_DZ_EVQ_RPTR, read_ptr & ptr_mask);
 130
 131        rte_write32_relaxed(dword.ed_u32[0], qprime);
 132        rte_wmb();
 133}
 134
 135
 136const uint32_t * sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps);
 137
 138
 139#ifdef __cplusplus
 140}
 141#endif
 142#endif /* _SFC_EF10_H */
 143