LXR linux/include/net/xdp.h

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/* include/net/xdp.h
   3 *
   4 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
   5 */
   6#ifndef __LINUX_NET_XDP_H__
   7#define __LINUX_NET_XDP_H__
   8
   9/**
  10 * DOC: XDP RX-queue information
  11 *
  12 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
  13 * level RX-ring queues.  It is information that is specific to how
  14 * the driver have configured a given RX-ring queue.
  15 *
  16 * Each xdp_buff frame received in the driver carry a (pointer)
  17 * reference to this xdp_rxq_info structure.  This provides the XDP
  18 * data-path read-access to RX-info for both kernel and bpf-side
  19 * (limited subset).
  20 *
  21 * For now, direct access is only safe while running in NAPI/softirq
  22 * context.  Contents is read-mostly and must not be updated during
  23 * driver NAPI/softirq poll.
  24 *
  25 * The driver usage API is a register and unregister API.
  26 *
  27 * The struct is not directly tied to the XDP prog.  A new XDP prog
  28 * can be attached as long as it doesn't change the underlying
  29 * RX-ring.  If the RX-ring does change significantly, the NIC driver
  30 * naturally need to stop the RX-ring before purging and reallocating
  31 * memory.  In that process the driver MUST call unregistor (which
  32 * also apply for driver shutdown and unload).  The register API is
  33 * also mandatory during RX-ring setup.
  34 */
  35
  36enum xdp_mem_type {
  37        MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
  38        MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */
  39        MEM_TYPE_PAGE_POOL,
  40        MEM_TYPE_ZERO_COPY,
  41        MEM_TYPE_MAX,
  42};
  43
  44/* XDP flags for ndo_xdp_xmit */
  45#define XDP_XMIT_FLUSH          (1U << 0)       /* doorbell signal consumer */
  46#define XDP_XMIT_FLAGS_MASK     XDP_XMIT_FLUSH
  47
  48struct xdp_mem_info {
  49        u32 type; /* enum xdp_mem_type, but known size type */
  50        u32 id;
  51};
  52
  53struct page_pool;
  54
  55struct zero_copy_allocator {
  56        void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
  57};
  58
  59struct xdp_rxq_info {
  60        struct net_device *dev;
  61        u32 queue_index;
  62        u32 reg_state;
  63        struct xdp_mem_info mem;
  64} ____cacheline_aligned; /* perf critical, avoid false-sharing */
  65
  66struct xdp_buff {
  67        void *data;
  68        void *data_end;
  69        void *data_meta;
  70        void *data_hard_start;
  71        unsigned long handle;
  72        struct xdp_rxq_info *rxq;
  73};
  74
  75struct xdp_frame {
  76        void *data;
  77        u16 len;
  78        u16 headroom;
  79        u16 metasize;
  80        /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
  81         * while mem info is valid on remote CPU.
  82         */
  83        struct xdp_mem_info mem;
  84        struct net_device *dev_rx; /* used by cpumap */
  85};
  86
  87/* Clear kernel pointers in xdp_frame */
  88static inline void xdp_scrub_frame(struct xdp_frame *frame)
  89{
  90        frame->data = NULL;
  91        frame->dev_rx = NULL;
  92}
  93
  94struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
  95
  96/* Convert xdp_buff to xdp_frame */
  97static inline
  98struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
  99{
 100        struct xdp_frame *xdp_frame;
 101        int metasize;
 102        int headroom;
 103
 104        if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
 105                return xdp_convert_zc_to_xdp_frame(xdp);
 106
 107        /* Assure headroom is available for storing info */
 108        headroom = xdp->data - xdp->data_hard_start;
 109        metasize = xdp->data - xdp->data_meta;
 110        metasize = metasize > 0 ? metasize : 0;
 111        if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
 112                return NULL;
 113
 114        /* Store info in top of packet */
 115        xdp_frame = xdp->data_hard_start;
 116
 117        xdp_frame->data = xdp->data;
 118        xdp_frame->len  = xdp->data_end - xdp->data;
 119        xdp_frame->headroom = headroom - sizeof(*xdp_frame);
 120        xdp_frame->metasize = metasize;
 121
 122        /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
 123        xdp_frame->mem = xdp->rxq->mem;
 124
 125        return xdp_frame;
 126}
 127
 128void xdp_return_frame(struct xdp_frame *xdpf);
 129void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
 130void xdp_return_buff(struct xdp_buff *xdp);
 131
 132int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
 133                     struct net_device *dev, u32 queue_index);
 134void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
 135void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
 136bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
 137int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
 138                               enum xdp_mem_type type, void *allocator);
 139void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq);
 140
 141/* Drivers not supporting XDP metadata can use this helper, which
 142 * rejects any room expansion for metadata as a result.
 143 */
 144static __always_inline void
 145xdp_set_data_meta_invalid(struct xdp_buff *xdp)
 146{
 147        xdp->data_meta = xdp->data + 1;
 148}
 149
 150static __always_inline bool
 151xdp_data_meta_unsupported(const struct xdp_buff *xdp)
 152{
 153        return unlikely(xdp->data_meta > xdp->data);
 154}
 155
 156struct xdp_attachment_info {
 157        struct bpf_prog *prog;
 158        u32 flags;
 159};
 160
 161struct netdev_bpf;
 162int xdp_attachment_query(struct xdp_attachment_info *info,
 163                         struct netdev_bpf *bpf);
 164bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
 165                             struct netdev_bpf *bpf);
 166void xdp_attachment_setup(struct xdp_attachment_info *info,
 167                          struct netdev_bpf *bpf);
 168
 169#endif /* __LINUX_NET_XDP_H__ */
 170