Branch data Line data Source code
1 : : /* SPDX-License-Identifier: GPL-2.0-only */ 2 : : /* include/net/xdp.h 3 : : * 4 : : * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. 5 : : */ 6 : : #ifndef __LINUX_NET_XDP_H__ 7 : : #define __LINUX_NET_XDP_H__ 8 : : 9 : : /** 10 : : * DOC: XDP RX-queue information 11 : : * 12 : : * The XDP RX-queue info (xdp_rxq_info) is associated with the driver 13 : : * level RX-ring queues. It is information that is specific to how 14 : : * the driver have configured a given RX-ring queue. 15 : : * 16 : : * Each xdp_buff frame received in the driver carry a (pointer) 17 : : * reference to this xdp_rxq_info structure. This provides the XDP 18 : : * data-path read-access to RX-info for both kernel and bpf-side 19 : : * (limited subset). 20 : : * 21 : : * For now, direct access is only safe while running in NAPI/softirq 22 : : * context. Contents is read-mostly and must not be updated during 23 : : * driver NAPI/softirq poll. 24 : : * 25 : : * The driver usage API is a register and unregister API. 26 : : * 27 : : * The struct is not directly tied to the XDP prog. A new XDP prog 28 : : * can be attached as long as it doesn't change the underlying 29 : : * RX-ring. If the RX-ring does change significantly, the NIC driver 30 : : * naturally need to stop the RX-ring before purging and reallocating 31 : : * memory. In that process the driver MUST call unregistor (which 32 : : * also apply for driver shutdown and unload). The register API is 33 : : * also mandatory during RX-ring setup. 34 : : */ 35 : : 36 : : enum xdp_mem_type { 37 : : MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ 38 : : MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ 39 : : MEM_TYPE_PAGE_POOL, 40 : : MEM_TYPE_ZERO_COPY, 41 : : MEM_TYPE_MAX, 42 : : }; 43 : : 44 : : /* XDP flags for ndo_xdp_xmit */ 45 : : #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ 46 : : #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH 47 : : 48 : : struct xdp_mem_info { 49 : : u32 type; /* enum xdp_mem_type, but known size type */ 50 : : u32 id; 51 : : }; 52 : : 53 : : struct page_pool; 54 : : 55 : : struct zero_copy_allocator { 56 : : void (*free)(struct zero_copy_allocator *zca, unsigned long handle); 57 : : }; 58 : : 59 : : struct xdp_rxq_info { 60 : : struct net_device *dev; 61 : : u32 queue_index; 62 : : u32 reg_state; 63 : : struct xdp_mem_info mem; 64 : : } ____cacheline_aligned; /* perf critical, avoid false-sharing */ 65 : : 66 : : struct xdp_buff { 67 : : void *data; 68 : : void *data_end; 69 : : void *data_meta; 70 : : void *data_hard_start; 71 : : unsigned long handle; 72 : : struct xdp_rxq_info *rxq; 73 : : }; 74 : : 75 : : struct xdp_frame { 76 : : void *data; 77 : : u16 len; 78 : : u16 headroom; 79 : : u16 metasize; 80 : : /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, 81 : : * while mem info is valid on remote CPU. 82 : : */ 83 : : struct xdp_mem_info mem; 84 : : struct net_device *dev_rx; /* used by cpumap */ 85 : : }; 86 : : 87 : : /* Clear kernel pointers in xdp_frame */ 88 : : static inline void xdp_scrub_frame(struct xdp_frame *frame) 89 : : { 90 : : frame->data = NULL; 91 : : frame->dev_rx = NULL; 92 : : } 93 : : 94 : : struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); 95 : : 96 : : /* Convert xdp_buff to xdp_frame */ 97 : : static inline 98 : 0 : struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) 99 : : { 100 : 0 : struct xdp_frame *xdp_frame; 101 : 0 : int metasize; 102 : 0 : int headroom; 103 : : 104 [ # # ]: 0 : if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) 105 : 0 : return xdp_convert_zc_to_xdp_frame(xdp); 106 : : 107 : : /* Assure headroom is available for storing info */ 108 : 0 : headroom = xdp->data - xdp->data_hard_start; 109 : 0 : metasize = xdp->data - xdp->data_meta; 110 : 0 : metasize = metasize > 0 ? metasize : 0; 111 [ # # ]: 0 : if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) 112 : : return NULL; 113 : : 114 : : /* Store info in top of packet */ 115 : 0 : xdp_frame = xdp->data_hard_start; 116 : : 117 : 0 : xdp_frame->data = xdp->data; 118 : 0 : xdp_frame->len = xdp->data_end - xdp->data; 119 : 0 : xdp_frame->headroom = headroom - sizeof(*xdp_frame); 120 : 0 : xdp_frame->metasize = metasize; 121 : : 122 : : /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ 123 : 0 : xdp_frame->mem = xdp->rxq->mem; 124 : : 125 : 0 : return xdp_frame; 126 : : } 127 : : 128 : : void xdp_return_frame(struct xdp_frame *xdpf); 129 : : void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); 130 : : void xdp_return_buff(struct xdp_buff *xdp); 131 : : 132 : : /* When sending xdp_frame into the network stack, then there is no 133 : : * return point callback, which is needed to release e.g. DMA-mapping 134 : : * resources with page_pool. Thus, have explicit function to release 135 : : * frame resources. 136 : : */ 137 : : void __xdp_release_frame(void *data, struct xdp_mem_info *mem); 138 : : static inline void xdp_release_frame(struct xdp_frame *xdpf) 139 : : { 140 : : struct xdp_mem_info *mem = &xdpf->mem; 141 : : 142 : : /* Curr only page_pool needs this */ 143 : : if (mem->type == MEM_TYPE_PAGE_POOL) 144 : : __xdp_release_frame(xdpf->data, mem); 145 : : } 146 : : 147 : : int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, 148 : : struct net_device *dev, u32 queue_index); 149 : : void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); 150 : : void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); 151 : : bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); 152 : : int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, 153 : : enum xdp_mem_type type, void *allocator); 154 : : void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); 155 : : 156 : : /* Drivers not supporting XDP metadata can use this helper, which 157 : : * rejects any room expansion for metadata as a result. 158 : : */ 159 : : static __always_inline void 160 : 0 : xdp_set_data_meta_invalid(struct xdp_buff *xdp) 161 : : { 162 [ # # # # ]: 0 : xdp->data_meta = xdp->data + 1; 163 : : } 164 : : 165 : : static __always_inline bool 166 : 0 : xdp_data_meta_unsupported(const struct xdp_buff *xdp) 167 : : { 168 [ # # # # ]: 0 : return unlikely(xdp->data_meta > xdp->data); 169 : : } 170 : : 171 : : struct xdp_attachment_info { 172 : : struct bpf_prog *prog; 173 : : u32 flags; 174 : : }; 175 : : 176 : : struct netdev_bpf; 177 : : int xdp_attachment_query(struct xdp_attachment_info *info, 178 : : struct netdev_bpf *bpf); 179 : : bool xdp_attachment_flags_ok(struct xdp_attachment_info *info, 180 : : struct netdev_bpf *bpf); 181 : : void xdp_attachment_setup(struct xdp_attachment_info *info, 182 : : struct netdev_bpf *bpf); 183 : : 184 : : #endif /* __LINUX_NET_XDP_H__ */