LXR linux/drivers/net/ethernet/freescale/enetc/enetc.h

   1/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
   2/* Copyright 2017-2019 NXP */
   3
   4#include <linux/timer.h>
   5#include <linux/pci.h>
   6#include <linux/netdevice.h>
   7#include <linux/etherdevice.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/skbuff.h>
  10#include <linux/ethtool.h>
  11#include <linux/if_vlan.h>
  12#include <linux/phylink.h>
  13#include <linux/dim.h>
  14
  15#include "enetc_hw.h"
  16
  17#define ENETC_MAC_MAXFRM_SIZE   9600
  18#define ENETC_MAX_MTU           (ENETC_MAC_MAXFRM_SIZE - \
  19                                (ETH_FCS_LEN + ETH_HLEN + VLAN_HLEN))
  20
  21struct enetc_tx_swbd {
  22        union {
  23                struct sk_buff *skb;
  24                struct xdp_frame *xdp_frame;
  25        };
  26        dma_addr_t dma;
  27        struct page *page;      /* valid only if is_xdp_tx */
  28        u16 page_offset;        /* valid only if is_xdp_tx */
  29        u16 len;
  30        enum dma_data_direction dir;
  31        u8 is_dma_page:1;
  32        u8 check_wb:1;
  33        u8 do_twostep_tstamp:1;
  34        u8 is_eof:1;
  35        u8 is_xdp_tx:1;
  36        u8 is_xdp_redirect:1;
  37};
  38
  39#define ENETC_RX_MAXFRM_SIZE    ENETC_MAC_MAXFRM_SIZE
  40#define ENETC_RXB_TRUESIZE      2048 /* PAGE_SIZE >> 1 */
  41#define ENETC_RXB_PAD           NET_SKB_PAD /* add extra space if needed */
  42#define ENETC_RXB_DMA_SIZE      \
  43        (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - ENETC_RXB_PAD)
  44#define ENETC_RXB_DMA_SIZE_XDP  \
  45        (SKB_WITH_OVERHEAD(ENETC_RXB_TRUESIZE) - XDP_PACKET_HEADROOM)
  46
  47struct enetc_rx_swbd {
  48        dma_addr_t dma;
  49        struct page *page;
  50        u16 page_offset;
  51        enum dma_data_direction dir;
  52        u16 len;
  53};
  54
  55/* ENETC overhead: optional extension BD + 1 BD gap */
  56#define ENETC_TXBDS_NEEDED(val) ((val) + 2)
  57/* max # of chained Tx BDs is 15, including head and extension BD */
  58#define ENETC_MAX_SKB_FRAGS     13
  59#define ENETC_TXBDS_MAX_NEEDED  ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
  60
  61struct enetc_ring_stats {
  62        unsigned int packets;
  63        unsigned int bytes;
  64        unsigned int rx_alloc_errs;
  65        unsigned int xdp_drops;
  66        unsigned int xdp_tx;
  67        unsigned int xdp_tx_drops;
  68        unsigned int xdp_redirect;
  69        unsigned int xdp_redirect_failures;
  70        unsigned int xdp_redirect_sg;
  71        unsigned int recycles;
  72        unsigned int recycle_failures;
  73};
  74
  75struct enetc_xdp_data {
  76        struct xdp_rxq_info rxq;
  77        struct bpf_prog *prog;
  78        int xdp_tx_in_flight;
  79};
  80
  81#define ENETC_RX_RING_DEFAULT_SIZE      2048
  82#define ENETC_TX_RING_DEFAULT_SIZE      2048
  83#define ENETC_DEFAULT_TX_WORK           (ENETC_TX_RING_DEFAULT_SIZE / 2)
  84
  85struct enetc_bdr {
  86        struct device *dev; /* for DMA mapping */
  87        struct net_device *ndev;
  88        void *bd_base; /* points to Rx or Tx BD ring */
  89        union {
  90                void __iomem *tpir;
  91                void __iomem *rcir;
  92        };
  93        u16 index;
  94        int bd_count; /* # of BDs */
  95        int next_to_use;
  96        int next_to_clean;
  97        union {
  98                struct enetc_tx_swbd *tx_swbd;
  99                struct enetc_rx_swbd *rx_swbd;
 100        };
 101        union {
 102                void __iomem *tcir; /* Tx */
 103                int next_to_alloc; /* Rx */
 104        };
 105        void __iomem *idr; /* Interrupt Detect Register pointer */
 106
 107        int buffer_offset;
 108        struct enetc_xdp_data xdp;
 109
 110        struct enetc_ring_stats stats;
 111
 112        dma_addr_t bd_dma_base;
 113        u8 tsd_enable; /* Time specific departure */
 114        bool ext_en; /* enable h/w descriptor extensions */
 115} ____cacheline_aligned_in_smp;
 116
 117static inline void enetc_bdr_idx_inc(struct enetc_bdr *bdr, int *i)
 118{
 119        if (unlikely(++*i == bdr->bd_count))
 120                *i = 0;
 121}
 122
 123static inline int enetc_bd_unused(struct enetc_bdr *bdr)
 124{
 125        if (bdr->next_to_clean > bdr->next_to_use)
 126                return bdr->next_to_clean - bdr->next_to_use - 1;
 127
 128        return bdr->bd_count + bdr->next_to_clean - bdr->next_to_use - 1;
 129}
 130
 131static inline int enetc_swbd_unused(struct enetc_bdr *bdr)
 132{
 133        if (bdr->next_to_clean > bdr->next_to_alloc)
 134                return bdr->next_to_clean - bdr->next_to_alloc - 1;
 135
 136        return bdr->bd_count + bdr->next_to_clean - bdr->next_to_alloc - 1;
 137}
 138
 139/* Control BD ring */
 140#define ENETC_CBDR_DEFAULT_SIZE 64
 141struct enetc_cbdr {
 142        void *bd_base; /* points to Rx or Tx BD ring */
 143        void __iomem *pir;
 144        void __iomem *cir;
 145        void __iomem *mr; /* mode register */
 146
 147        int bd_count; /* # of BDs */
 148        int next_to_use;
 149        int next_to_clean;
 150
 151        dma_addr_t bd_dma_base;
 152        struct device *dma_dev;
 153};
 154
 155#define ENETC_TXBD(BDR, i) (&(((union enetc_tx_bd *)((BDR).bd_base))[i]))
 156
 157static inline union enetc_rx_bd *enetc_rxbd(struct enetc_bdr *rx_ring, int i)
 158{
 159        int hw_idx = i;
 160
 161#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 162        if (rx_ring->ext_en)
 163                hw_idx = 2 * i;
 164#endif
 165        return &(((union enetc_rx_bd *)rx_ring->bd_base)[hw_idx]);
 166}
 167
 168static inline void enetc_rxbd_next(struct enetc_bdr *rx_ring,
 169                                   union enetc_rx_bd **old_rxbd, int *old_index)
 170{
 171        union enetc_rx_bd *new_rxbd = *old_rxbd;
 172        int new_index = *old_index;
 173
 174        new_rxbd++;
 175
 176#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 177        if (rx_ring->ext_en)
 178                new_rxbd++;
 179#endif
 180
 181        if (unlikely(++new_index == rx_ring->bd_count)) {
 182                new_rxbd = rx_ring->bd_base;
 183                new_index = 0;
 184        }
 185
 186        *old_rxbd = new_rxbd;
 187        *old_index = new_index;
 188}
 189
 190static inline union enetc_rx_bd *enetc_rxbd_ext(union enetc_rx_bd *rxbd)
 191{
 192        return ++rxbd;
 193}
 194
 195struct enetc_msg_swbd {
 196        void *vaddr;
 197        dma_addr_t dma;
 198        int size;
 199};
 200
 201#define ENETC_REV1      0x1
 202enum enetc_errata {
 203        ENETC_ERR_VLAN_ISOL     = BIT(0),
 204        ENETC_ERR_UCMCSWP       = BIT(1),
 205};
 206
 207#define ENETC_SI_F_QBV BIT(0)
 208#define ENETC_SI_F_PSFP BIT(1)
 209
 210/* PCI IEP device data */
 211struct enetc_si {
 212        struct pci_dev *pdev;
 213        struct enetc_hw hw;
 214        enum enetc_errata errata;
 215
 216        struct net_device *ndev; /* back ref. */
 217
 218        struct enetc_cbdr cbd_ring;
 219
 220        int num_rx_rings; /* how many rings are available in the SI */
 221        int num_tx_rings;
 222        int num_fs_entries;
 223        int num_rss; /* number of RSS buckets */
 224        unsigned short pad;
 225        int hw_features;
 226};
 227
 228#define ENETC_SI_ALIGN  32
 229
 230static inline void *enetc_si_priv(const struct enetc_si *si)
 231{
 232        return (char *)si + ALIGN(sizeof(struct enetc_si), ENETC_SI_ALIGN);
 233}
 234
 235static inline bool enetc_si_is_pf(struct enetc_si *si)
 236{
 237        return !!(si->hw.port);
 238}
 239
 240static inline int enetc_pf_to_port(struct pci_dev *pf_pdev)
 241{
 242        switch (pf_pdev->devfn) {
 243        case 0:
 244                return 0;
 245        case 1:
 246                return 1;
 247        case 2:
 248                return 2;
 249        case 6:
 250                return 3;
 251        default:
 252                return -1;
 253        }
 254}
 255
 256#define ENETC_MAX_NUM_TXQS      8
 257#define ENETC_INT_NAME_MAX      (IFNAMSIZ + 8)
 258
 259struct enetc_int_vector {
 260        void __iomem *rbier;
 261        void __iomem *tbier_base;
 262        void __iomem *ricr1;
 263        unsigned long tx_rings_map;
 264        int count_tx_rings;
 265        u32 rx_ictt;
 266        u16 comp_cnt;
 267        bool rx_dim_en, rx_napi_work;
 268        struct napi_struct napi ____cacheline_aligned_in_smp;
 269        struct dim rx_dim ____cacheline_aligned_in_smp;
 270        char name[ENETC_INT_NAME_MAX];
 271
 272        struct enetc_bdr rx_ring;
 273        struct enetc_bdr tx_ring[];
 274} ____cacheline_aligned_in_smp;
 275
 276struct enetc_cls_rule {
 277        struct ethtool_rx_flow_spec fs;
 278        int used;
 279};
 280
 281#define ENETC_MAX_BDR_INT       2 /* fixed to max # of available cpus */
 282struct psfp_cap {
 283        u32 max_streamid;
 284        u32 max_psfp_filter;
 285        u32 max_psfp_gate;
 286        u32 max_psfp_gatelist;
 287        u32 max_psfp_meter;
 288};
 289
 290#define ENETC_F_TX_TSTAMP_MASK  0xff
 291/* TODO: more hardware offloads */
 292enum enetc_active_offloads {
 293        /* 8 bits reserved for TX timestamp types (hwtstamp_tx_types) */
 294        ENETC_F_TX_TSTAMP               = BIT(0),
 295        ENETC_F_TX_ONESTEP_SYNC_TSTAMP  = BIT(1),
 296
 297        ENETC_F_RX_TSTAMP               = BIT(8),
 298        ENETC_F_QBV                     = BIT(9),
 299        ENETC_F_QCI                     = BIT(10),
 300};
 301
 302enum enetc_flags_bit {
 303        ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS = 0,
 304};
 305
 306/* interrupt coalescing modes */
 307enum enetc_ic_mode {
 308        /* one interrupt per frame */
 309        ENETC_IC_NONE = 0,
 310        /* activated when int coalescing time is set to a non-0 value */
 311        ENETC_IC_RX_MANUAL = BIT(0),
 312        ENETC_IC_TX_MANUAL = BIT(1),
 313        /* use dynamic interrupt moderation */
 314        ENETC_IC_RX_ADAPTIVE = BIT(2),
 315};
 316
 317#define ENETC_RXIC_PKTTHR       min_t(u32, 256, ENETC_RX_RING_DEFAULT_SIZE / 2)
 318#define ENETC_TXIC_PKTTHR       min_t(u32, 128, ENETC_TX_RING_DEFAULT_SIZE / 2)
 319#define ENETC_TXIC_TIMETHR      enetc_usecs_to_cycles(600)
 320
 321struct enetc_ndev_priv {
 322        struct net_device *ndev;
 323        struct device *dev; /* dma-mapping device */
 324        struct enetc_si *si;
 325
 326        int bdr_int_num; /* number of Rx/Tx ring interrupts */
 327        struct enetc_int_vector *int_vector[ENETC_MAX_BDR_INT];
 328        u16 num_rx_rings, num_tx_rings;
 329        u16 rx_bd_count, tx_bd_count;
 330
 331        u16 msg_enable;
 332        enum enetc_active_offloads active_offloads;
 333
 334        u32 speed; /* store speed for compare update pspeed */
 335
 336        struct enetc_bdr **xdp_tx_ring;
 337        struct enetc_bdr *tx_ring[16];
 338        struct enetc_bdr *rx_ring[16];
 339
 340        struct enetc_cls_rule *cls_rules;
 341
 342        struct psfp_cap psfp_cap;
 343
 344        struct phylink *phylink;
 345        int ic_mode;
 346        u32 tx_ictt;
 347
 348        struct bpf_prog *xdp_prog;
 349
 350        unsigned long flags;
 351
 352        struct work_struct      tx_onestep_tstamp;
 353        struct sk_buff_head     tx_skbs;
 354};
 355
 356/* Messaging */
 357
 358/* VF-PF set primary MAC address message format */
 359struct enetc_msg_cmd_set_primary_mac {
 360        struct enetc_msg_cmd_header header;
 361        struct sockaddr mac;
 362};
 363
 364#define ENETC_CBD(R, i) (&(((struct enetc_cbd *)((R).bd_base))[i]))
 365
 366#define ENETC_CBDR_TIMEOUT      1000 /* usecs */
 367
 368/* PTP driver exports */
 369extern int enetc_phc_index;
 370
 371/* SI common */
 372int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv);
 373void enetc_pci_remove(struct pci_dev *pdev);
 374int enetc_alloc_msix(struct enetc_ndev_priv *priv);
 375void enetc_free_msix(struct enetc_ndev_priv *priv);
 376void enetc_get_si_caps(struct enetc_si *si);
 377void enetc_init_si_rings_params(struct enetc_ndev_priv *priv);
 378int enetc_alloc_si_resources(struct enetc_ndev_priv *priv);
 379void enetc_free_si_resources(struct enetc_ndev_priv *priv);
 380int enetc_configure_si(struct enetc_ndev_priv *priv);
 381
 382int enetc_open(struct net_device *ndev);
 383int enetc_close(struct net_device *ndev);
 384void enetc_start(struct net_device *ndev);
 385void enetc_stop(struct net_device *ndev);
 386netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev);
 387struct net_device_stats *enetc_get_stats(struct net_device *ndev);
 388int enetc_set_features(struct net_device *ndev,
 389                       netdev_features_t features);
 390int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd);
 391int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
 392                   void *type_data);
 393int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp);
 394int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
 395                   struct xdp_frame **frames, u32 flags);
 396
 397/* ethtool */
 398void enetc_set_ethtool_ops(struct net_device *ndev);
 399
 400/* control buffer descriptor ring (CBDR) */
 401int enetc_setup_cbdr(struct device *dev, struct enetc_hw *hw, int bd_count,
 402                     struct enetc_cbdr *cbdr);
 403void enetc_teardown_cbdr(struct enetc_cbdr *cbdr);
 404int enetc_set_mac_flt_entry(struct enetc_si *si, int index,
 405                            char *mac_addr, int si_map);
 406int enetc_clear_mac_flt_entry(struct enetc_si *si, int index);
 407int enetc_set_fs_entry(struct enetc_si *si, struct enetc_cmd_rfse *rfse,
 408                       int index);
 409void enetc_set_rss_key(struct enetc_hw *hw, const u8 *bytes);
 410int enetc_get_rss_table(struct enetc_si *si, u32 *table, int count);
 411int enetc_set_rss_table(struct enetc_si *si, const u32 *table, int count);
 412int enetc_send_cmd(struct enetc_si *si, struct enetc_cbd *cbd);
 413
 414#ifdef CONFIG_FSL_ENETC_QOS
 415int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data);
 416void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed);
 417int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data);
 418int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data);
 419int enetc_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 420                            void *cb_priv);
 421int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data);
 422int enetc_psfp_init(struct enetc_ndev_priv *priv);
 423int enetc_psfp_clean(struct enetc_ndev_priv *priv);
 424
 425static inline void enetc_get_max_cap(struct enetc_ndev_priv *priv)
 426{
 427        u32 reg;
 428
 429        reg = enetc_port_rd(&priv->si->hw, ENETC_PSIDCAPR);
 430        priv->psfp_cap.max_streamid = reg & ENETC_PSIDCAPR_MSK;
 431        /* Port stream filter capability */
 432        reg = enetc_port_rd(&priv->si->hw, ENETC_PSFCAPR);
 433        priv->psfp_cap.max_psfp_filter = reg & ENETC_PSFCAPR_MSK;
 434        /* Port stream gate capability */
 435        reg = enetc_port_rd(&priv->si->hw, ENETC_PSGCAPR);
 436        priv->psfp_cap.max_psfp_gate = (reg & ENETC_PSGCAPR_SGIT_MSK);
 437        priv->psfp_cap.max_psfp_gatelist = (reg & ENETC_PSGCAPR_GCL_MSK) >> 16;
 438        /* Port flow meter capability */
 439        reg = enetc_port_rd(&priv->si->hw, ENETC_PFMCAPR);
 440        priv->psfp_cap.max_psfp_meter = reg & ENETC_PFMCAPR_MSK;
 441}
 442
 443static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
 444{
 445        struct enetc_hw *hw = &priv->si->hw;
 446        int err;
 447
 448        enetc_get_max_cap(priv);
 449
 450        err = enetc_psfp_init(priv);
 451        if (err)
 452                return err;
 453
 454        enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) |
 455                 ENETC_PPSFPMR_PSFPEN | ENETC_PPSFPMR_VS |
 456                 ENETC_PPSFPMR_PVC | ENETC_PPSFPMR_PVZC);
 457
 458        return 0;
 459}
 460
 461static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
 462{
 463        struct enetc_hw *hw = &priv->si->hw;
 464        int err;
 465
 466        err = enetc_psfp_clean(priv);
 467        if (err)
 468                return err;
 469
 470        enetc_wr(hw, ENETC_PPSFPMR, enetc_rd(hw, ENETC_PPSFPMR) &
 471                 ~ENETC_PPSFPMR_PSFPEN & ~ENETC_PPSFPMR_VS &
 472                 ~ENETC_PPSFPMR_PVC & ~ENETC_PPSFPMR_PVZC);
 473
 474        memset(&priv->psfp_cap, 0, sizeof(struct psfp_cap));
 475
 476        return 0;
 477}
 478
 479#else
 480#define enetc_setup_tc_taprio(ndev, type_data) -EOPNOTSUPP
 481#define enetc_sched_speed_set(priv, speed) (void)0
 482#define enetc_setup_tc_cbs(ndev, type_data) -EOPNOTSUPP
 483#define enetc_setup_tc_txtime(ndev, type_data) -EOPNOTSUPP
 484#define enetc_setup_tc_psfp(ndev, type_data) -EOPNOTSUPP
 485#define enetc_setup_tc_block_cb NULL
 486
 487#define enetc_get_max_cap(p)            \
 488        memset(&((p)->psfp_cap), 0, sizeof(struct psfp_cap))
 489
 490static inline int enetc_psfp_enable(struct enetc_ndev_priv *priv)
 491{
 492        return 0;
 493}
 494
 495static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
 496{
 497        return 0;
 498}
 499#endif
 500