LXR linux/include/net/xdp.h

   1/* include/net/xdp.h
   2 *
   3 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
   4 * Released under terms in GPL version 2.  See COPYING.
   5 */
   6#ifndef __LINUX_NET_XDP_H__
   7#define __LINUX_NET_XDP_H__
   8
   9/**
  10 * DOC: XDP RX-queue information
  11 *
  12 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
  13 * level RX-ring queues.  It is information that is specific to how
  14 * the driver have configured a given RX-ring queue.
  15 *
  16 * Each xdp_buff frame received in the driver carry a (pointer)
  17 * reference to this xdp_rxq_info structure.  This provides the XDP
  18 * data-path read-access to RX-info for both kernel and bpf-side
  19 * (limited subset).
  20 *
  21 * For now, direct access is only safe while running in NAPI/softirq
  22 * context.  Contents is read-mostly and must not be updated during
  23 * driver NAPI/softirq poll.
  24 *
  25 * The driver usage API is a register and unregister API.
  26 *
  27 * The struct is not directly tied to the XDP prog.  A new XDP prog
  28 * can be attached as long as it doesn't change the underlying
  29 * RX-ring.  If the RX-ring does change significantly, the NIC driver
  30 * naturally need to stop the RX-ring before purging and reallocating
  31 * memory.  In that process the driver MUST call unregistor (which
  32 * also apply for driver shutdown and unload).  The register API is
  33 * also mandatory during RX-ring setup.
  34 */
  35
  36enum xdp_mem_type {
  37        MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
  38        MEM_TYPE_PAGE_ORDER0,     /* Orig XDP full page model */
  39        MEM_TYPE_PAGE_POOL,
  40        MEM_TYPE_ZERO_COPY,
  41        MEM_TYPE_MAX,
  42};
  43
  44/* XDP flags for ndo_xdp_xmit */
  45#define XDP_XMIT_FLUSH          (1U << 0)       /* doorbell signal consumer */
  46#define XDP_XMIT_FLAGS_MASK     XDP_XMIT_FLUSH
  47
  48struct xdp_mem_info {
  49        u32 type; /* enum xdp_mem_type, but known size type */
  50        u32 id;
  51};
  52
  53struct page_pool;
  54
  55struct zero_copy_allocator {
  56        void (*free)(struct zero_copy_allocator *zca, unsigned long handle);
  57};
  58
  59struct xdp_rxq_info {
  60        struct net_device *dev;
  61        u32 queue_index;
  62        u32 reg_state;
  63        struct xdp_mem_info mem;
  64} ____cacheline_aligned; /* perf critical, avoid false-sharing */
  65
  66struct xdp_buff {
  67        void *data;
  68        void *data_end;
  69        void *data_meta;
  70        void *data_hard_start;
  71        unsigned long handle;
  72        struct xdp_rxq_info *rxq;
  73};
  74
  75struct xdp_frame {
  76        void *data;
  77        u16 len;
  78        u16 headroom;
  79        u16 metasize;
  80        /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
  81         * while mem info is valid on remote CPU.
  82         */
  83        struct xdp_mem_info mem;
  84        struct net_device *dev_rx; /* used by cpumap */
  85};
  86
  87/* Convert xdp_buff to xdp_frame */
  88static inline
  89struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
  90{
  91        struct xdp_frame *xdp_frame;
  92        int metasize;
  93        int headroom;
  94
  95        /* TODO: implement clone, copy, use "native" MEM_TYPE */
  96        if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
  97                return NULL;
  98
  99        /* Assure headroom is available for storing info */
 100        headroom = xdp->data - xdp->data_hard_start;
 101        metasize = xdp->data - xdp->data_meta;
 102        metasize = metasize > 0 ? metasize : 0;
 103        if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
 104                return NULL;
 105
 106        /* Store info in top of packet */
 107        xdp_frame = xdp->data_hard_start;
 108
 109        xdp_frame->data = xdp->data;
 110        xdp_frame->len  = xdp->data_end - xdp->data;
 111        xdp_frame->headroom = headroom - sizeof(*xdp_frame);
 112        xdp_frame->metasize = metasize;
 113
 114        /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
 115        xdp_frame->mem = xdp->rxq->mem;
 116
 117        return xdp_frame;
 118}
 119
 120void xdp_return_frame(struct xdp_frame *xdpf);
 121void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
 122void xdp_return_buff(struct xdp_buff *xdp);
 123
 124int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
 125                     struct net_device *dev, u32 queue_index);
 126void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
 127void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
 128bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
 129int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
 130                               enum xdp_mem_type type, void *allocator);
 131
 132/* Drivers not supporting XDP metadata can use this helper, which
 133 * rejects any room expansion for metadata as a result.
 134 */
 135static __always_inline void
 136xdp_set_data_meta_invalid(struct xdp_buff *xdp)
 137{
 138        xdp->data_meta = xdp->data + 1;
 139}
 140
 141static __always_inline bool
 142xdp_data_meta_unsupported(const struct xdp_buff *xdp)
 143{
 144        return unlikely(xdp->data_meta > xdp->data);
 145}
 146
 147#endif /* __LINUX_NET_XDP_H__ */
 148