| /linux/include/net/libeth/ |
| H A D | xsk.h | 7 #include <net/libeth/xdp.h>
19 * @bq: XDP Tx bulk to queue the head frag to
20 * @xdp: XSk buffer with the head to queue
25 struct libeth_xdp_buff *xdp) in libeth_xsk_tx_queue_head() argument
28 .xsk = xdp, in libeth_xsk_tx_queue_head()
29 __libeth_xdp_tx_len(xdp->base.data_end - xdp->data, in libeth_xsk_tx_queue_head()
33 if (likely(!xdp_buff_has_frags(&xdp->base))) in libeth_xsk_tx_queue_head()
43 * @bq: XDP Tx bulk to queue the frag to
57 * @bq: XDP Tx bulk to queue the frame to
58 * @xdp: XSk buffer to queue
[all …]
|
| H A D | xdp.h | 43 * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
58 offsetof(struct xdp_buff_xsk, xdp.data));
114 * Return: number of XDP Tx queues the device needs to use.
317 * struct libeth_xdp_tx_frame - represents one XDP Tx element
324 * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
368 * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
369 * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
372 * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
376 * All XDP Tx operations except XSk xmit queue each frame to the bulk first
434 * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
[all …]
|
| /linux/include/net/ |
| H A D | xdp_sock_drv.h | 67 * matching the XDP core's xdp_data_hard_end() layout. in xsk_pool_get_rx_frame_size()
106 static inline dma_addr_t xsk_buff_xdp_get_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_dma() argument
108 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma()
113 static inline dma_addr_t xsk_buff_xdp_get_frame_dma(struct xdp_buff *xdp) in xsk_buff_xdp_get_frame_dma() argument
115 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma()
131 static inline u32 xsk_buff_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max) in xsk_buff_alloc_batch() argument
133 return xp_alloc_batch(pool, xdp, max); in xsk_buff_alloc_batch()
141 static inline void xsk_buff_free(struct xdp_buff *xdp) in xsk_buff_free() argument
143 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free()
147 if (likely(!xdp_buff_has_frags(xdp))) in xsk_buff_free()
[all …]
|
| /linux/drivers/net/ethernet/broadcom/bnxt/ |
| H A D | bnxt_xdp.c | 29 struct xdp_buff *xdp) in bnxt_xmit_bd() argument
39 if (xdp && xdp_buff_has_frags(xdp)) { in bnxt_xmit_bd()
40 sinfo = xdp_get_shared_info_from_buff(xdp); in bnxt_xmit_bd()
48 if (xdp) in bnxt_xmit_bd()
49 tx_buf->page = virt_to_head_page(xdp->data); in bnxt_xmit_bd()
97 struct xdp_buff *xdp) in __bnxt_xmit_xdp() argument
101 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); in __bnxt_xmit_xdp()
184 struct xdp_buff *xdp) in bnxt_xdp_buff_init() argument
199 xdp_init_buff(xdp, buflen, &rxr->xdp_rxq); in bnxt_xdp_buff_init()
200 xdp_prepare_buff(xdp, data_ptr - offset, offset, len, true); in bnxt_xdp_buff_init()
[all …]
|
| /linux/tools/testing/selftests/bpf/progs/ |
| H A D | xdp_features.c | 65 xdp_process_echo_packet(struct xdp_md *xdp, bool dut) in xdp_process_echo_packet() argument
67 void *data_end = (void *)(long)xdp->data_end; in xdp_process_echo_packet()
68 void *data = (void *)(long)xdp->data; in xdp_process_echo_packet()
135 xdp_update_stats(struct xdp_md *xdp, bool tx, bool dut) in xdp_update_stats() argument
139 if (xdp_process_echo_packet(xdp, tx)) in xdp_update_stats()
155 SEC("xdp")
156 int xdp_tester_check_tx(struct xdp_md *xdp) in xdp_tester_check_tx() argument
158 xdp_update_stats(xdp, true, false); in xdp_tester_check_tx()
163 SEC("xdp")
164 int xdp_tester_check_rx(struct xdp_md *xdp) in xdp_tester_check_rx() argument
[all …]
|
| H A D | verifier_xdp_direct_packet_access.c | 8 SEC("xdp")
9 __description("XDP pkt read, pkt_end mangling, bad access 1")
29 SEC("xdp")
30 __description("XDP pkt read, pkt_end mangling, bad access 2")
50 SEC("xdp")
51 __description("XDP pkt read, pkt_data' > pkt_end, corner case, good access")
70 SEC("xdp")
71 __description("XDP pkt read, pkt_data' > pkt_end, bad access 1")
91 SEC("xdp")
92 __description("XDP pk
[all...] |
| H A D | xsk_xdp_progs.c | 22 SEC("xdp.frags") int xsk_def_prog(struct xdp_md *xdp) in xsk_def_prog() argument
27 SEC("xdp.frags") int xsk_xdp_drop(struct xdp_md *xdp) in xsk_xdp_drop() argument
38 SEC("xdp.frags") int xsk_xdp_populate_metadata(struct xdp_md *xdp) in xsk_xdp_populate_metadata() argument
45 err = bpf_xdp_adjust_meta(xdp, -(int)sizeof(struct xdp_info)); in xsk_xdp_populate_metadata()
49 data = (void *)(long)xdp->data; in xsk_xdp_populate_metadata()
50 data_meta = (void *)(long)xdp->data_meta; in xsk_xdp_populate_metadata()
61 SEC("xdp") int xsk_xdp_shared_umem(struct xdp_md *xdp) in xsk_xdp_shared_umem() argument
63 void *data = (void *)(long)xdp->data; in xsk_xdp_shared_umem()
64 void *data_end = (void *)(long)xdp->data_end; in xsk_xdp_shared_umem()
78 SEC("xdp.frags") int xsk_xdp_adjust_tail(struct xdp_md *xdp) in xsk_xdp_adjust_tail() argument
[all …]
|
| H A D | test_xdp_do_redirect.c | 11 * @MARK_IN: frame is being processed by the input XDP prog.
28 SEC("xdp")
29 int xdp_redirect(struct xdp_md *xdp) in xdp_redirect() argument
31 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_redirect()
32 void *data_end = (void *)(long)xdp->data_end; in xdp_redirect()
33 void *data = (void *)(long)xdp->data; in xdp_redirect()
41 if (xdp->ingress_ifindex != (__u32)ifindex_in) in xdp_redirect()
55 if (bpf_xdp_adjust_meta(xdp, sizeof(__u64))) in xdp_redirect()
85 SEC("xdp")
86 int xdp_count_pkts(struct xdp_md *xdp) in xdp_count_pkts() argument
[all …]
|
| H A D | test_xdp.c | 79 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
81 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
82 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
182 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| H A D | test_xdp_loop.c | 75 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
77 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
78 void *data = (void *)(long)xdp->data; in handle_ipv4()
108 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
111 data = (void *)(long)xdp->data; in handle_ipv4()
112 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
148 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
150 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
151 void *data = (void *)(long)xdp->data; in handle_ipv6()
178 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| /linux/Documentation/bpf/ |
| H A D | redirect.rst | 25 :doc: xdp redirect
29 those that do, not all of them support non-linear frames. Non-linear xdp
41 The following bpftrace command can be used to capture and count all XDP tracepoints:
45 sudo bpftrace -e 'tracepoint:xdp:* { @cnt[probe] = count(); }'
49 @cnt[tracepoint:xdp:mem_connect]: 18
50 @cnt[tracepoint:xdp:mem_disconnect]: 18
51 @cnt[tracepoint:xdp:xdp_exception]: 19605
52 @cnt[tracepoint:xdp:xdp_devmap_xmit]: 1393604
53 @cnt[tracepoint:xdp:xdp_redirect]: 22292200
56 The various xdp tracepoints can be found in ``source/include/trace/events/xdp.h``
[all …]
|
| /linux/net/core/ |
| H A D | xdp.c | 2 /* net/core/xdp.c
21 #include <net/xdp.h>
23 #include <trace/events/xdp.h>
372 * xdp_reg_page_pool - register &page_pool as a memory provider for XDP
375 * Can be used to register pools manually without connecting to any XDP RxQ
376 * info, so that the XDP layer will be aware of them. Then, they can be
409 * @xdp_rxq: XDP RxQ info to attach the pool to
427 /* XDP RX runs under NAPI protection, and in different delivery error
434 bool napi_direct, struct xdp_buff *xdp) in __xdp_return() argument
455 xsk_buff_free(xdp); in __xdp_return()
[all …]
|
| /linux/drivers/net/ethernet/intel/libeth/ |
| H A D | xdp.c | 8 #include <net/libeth/xdp.h>
29 netdev_warn(dev, "XDPSQ sharing enabled, possible XDP Tx slowdown\n"); in __libeth_xdpsq_get()
114 * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames
115 * @bq: XDP Tx frame bulk
120 * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust
205 * @src: source XDP buffer
208 * Use the fields from the passed XDP buffer to initialize the stash on the
228 LIBETH_XDP_ONSTACK_BUFF(xdp); in __libeth_xdp_return_stash()
230 libeth_xdp_load_stash(xdp, stash); in __libeth_xdp_return_stash()
231 libeth_xdp_return_buff_slow(xdp); in __libeth_xdp_return_stash()
[all …]
|
| H A D | xsk.c | 31 * @xdp: buffer to free
36 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp) in libeth_xsk_buff_free_slow() argument
38 xsk_buff_free(&xdp->base); in libeth_xsk_buff_free_slow()
45 * @xdp: frag buffer
54 struct libeth_xdp_buff *xdp) in libeth_xsk_buff_add_frag() argument
56 if (!xsk_buff_add_frag(&head->base, &xdp->base)) in libeth_xsk_buff_add_frag()
62 libeth_xsk_buff_free_slow(xdp); in libeth_xsk_buff_add_frag()
72 * @xdp: buffer to account
78 const struct libeth_xdp_buff *xdp) in libeth_xsk_buff_stats_frags() argument
80 libeth_xdp_buff_stats_frags(rs, xdp); in libeth_xsk_buff_stats_frags()
[all …]
|
| /linux/Documentation/networking/ |
| H A D | xdp-rx-metadata.rst | 4 XDP RX Metadata
7 This document describes how an eXpress Data Path (XDP) program can access
14 XDP has access to a set of kfuncs to manipulate the metadata in an XDP frame.
16 implement these kfuncs. The set of kfuncs is declared in ``include/net/xdp.h``
22 .. kernel-doc:: net/core/xdp.c
25 .. kernel-doc:: net/core/xdp.c
28 .. kernel-doc:: net/core/xdp.c
31 An XDP program can use these kfuncs to read the metadata into stack
33 consumers, an XDP progra
[all...] |
| /linux/drivers/net/vmxnet3/ |
| H A D | vmxnet3_xdp.c | 48 NL_SET_ERR_MSG_FMT_MOD(extack, "MTU %u too large for XDP", in vmxnet3_xdp_set()
54 NL_SET_ERR_MSG_MOD(extack, "LRO is not supported with XDP"); in vmxnet3_xdp_set()
86 "failed to re-create rx queues for XDP."); in vmxnet3_xdp_set()
92 "failed to activate device for XDP."); in vmxnet3_xdp_set()
100 /* This is the main xdp call used by kernel to set/unset eBPF program. */
152 } else { /* XDP buffer from page pool */ in vmxnet3_xdp_xmit_frame()
261 vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct xdp_buff *xdp, in vmxnet3_run_xdp() argument
270 act = bpf_prog_run_xdp(prog, xdp); in vmxnet3_run_xdp()
271 page = virt_to_page(xdp->data_hard_start); in vmxnet3_run_xdp()
277 err = xdp_do_redirect(rq->adapter->netdev, xdp, prog); in vmxnet3_run_xdp()
[all …]
|
| /linux/drivers/net/ethernet/intel/igb/ |
| H A D | igb_xsk.c | 6 #include <net/xdp.h>
176 static u16 igb_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in igb_fill_rx_descs() argument
187 buffs = xsk_buff_alloc_batch(pool, xdp, count); in igb_fill_rx_descs()
189 dma = xsk_buff_xdp_get_dma(*xdp); in igb_fill_rx_descs()
194 xdp++; in igb_fill_rx_descs()
207 struct xdp_buff **xdp; in igb_alloc_rx_buffers_zc() local
210 xdp = &rx_ring->rx_buffer_info_zc[ntu]; in igb_alloc_rx_buffers_zc()
213 nb_buffs_extra = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, in igb_alloc_rx_buffers_zc()
220 xdp = rx_ring->rx_buffer_info_zc; in igb_alloc_rx_buffers_zc()
225 nb_buffs = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, count); in igb_alloc_rx_buffers_zc()
[all …]
|
| /linux/drivers/net/ethernet/intel/ice/ |
| H A D | ice_xsk.c | 6 #include <net/libeth/xdp.h>
8 #include <net/xdp.h>
249 * @xdp: SW ring of xdp_buff that will hold the buffers
260 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in ice_fill_rx_descs() argument
267 buffs = xsk_buff_alloc_batch(pool, xdp, count); in ice_fill_rx_descs()
269 dma = xsk_buff_xdp_get_dma(*xdp); in ice_fill_rx_descs()
274 xdp++; in ice_fill_rx_descs()
299 struct xdp_buff **xdp; in __ice_alloc_rx_bufs_zc() local
302 xdp = ice_xdp_buf(rx_ring, ntu); in __ice_alloc_rx_bufs_zc()
305 nb_buffs_extra = ice_fill_rx_descs(xsk_pool, xdp, rx_desc, in __ice_alloc_rx_bufs_zc()
[all …]
|
| /linux/drivers/net/hyperv/ |
| H A D | netvsc_bpf.c | 17 #include <net/xdp.h>
25 struct xdp_buff *xdp) in netvsc_run_xdp() argument
35 xdp->data_hard_start = NULL; in netvsc_run_xdp()
56 xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); in netvsc_run_xdp()
57 xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); in netvsc_run_xdp()
59 memcpy(xdp->data, data, len); in netvsc_run_xdp()
61 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
73 if (!xdp_do_redirect(ndev, xdp, prog)) { in netvsc_run_xdp()
107 xdp->data_hard_start = NULL; in netvsc_run_xdp()
138 netdev_err(dev, "XDP: mtu:%u too large, buf_max:%u\n", in netvsc_xdp_set()
[all …]
|
| /linux/tools/testing/selftests/drivers/net/ |
| H A D | xdp.py | 5 This file contains tests to verify native XDP support in network drivers.
7 directory, with each test focusing on a specific aspect of XDP functionality.
24 """Enum for XDP configuration options."""
32 """Enum for XDP actions."""
41 """Enum for XDP statistics."""
54 xdp_sec: str = "xdp" # XDP section name (e.g., "xdp" or "xdp.frags")
102 Loads an XDP progra
[all...] |
| /linux/drivers/net/ethernet/microchip/lan966x/ |
| H A D | lan966x_xdp.c | 9 static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp_setup() argument
18 NL_SET_ERR_MSG_MOD(xdp->extack, in lan966x_xdp_setup()
19 "Allow to set xdp only when using fdma"); in lan966x_xdp_setup()
24 old_prog = xchg(&port->xdp_prog, xdp->prog); in lan966x_xdp_setup()
43 int lan966x_xdp(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp() argument
45 switch (xdp->command) { in lan966x_xdp()
47 return lan966x_xdp_setup(dev, xdp); in lan966x_xdp()
79 struct xdp_buff xdp; in lan966x_xdp_run() local
82 xdp_init_buff(&xdp, PAGE_SIZE << lan966x->rx.page_order, in lan966x_xdp_run()
84 xdp_prepare_buff(&xdp, page_address(page), in lan966x_xdp_run()
[all …]
|
| /linux/samples/bpf/ |
| H A D | xdp_tx_iptunnel_kern.c | 77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| /linux/tools/bpf/bpftool/Documentation/ |
| H A D | bpftool-net.rst | 32 | *ATTACH_TYPE* := { **xdp** | **xdpgeneric** | **xdpdrv** | **xdpoffload** | **tcx_ingress** | **tcx_egress** }
39 Currently, device driver xdp attachments, tcx, netkit and old-style tc
52 The current output will start with all xdp program attachments, followed by
54 flow_dissector and finally netfilter programs. Both xdp programs and
66 **xdp** - try native XDP and fallback to generic XDP if NIC driver does not support it;
67 **xdpgeneric** - Generic XDP. runs at generic XDP hook when packet already enters receive path as skb;
68 **xdpdrv** - Native XDP
[all...] |
| /linux/include/trace/events/ |
| H A D | xdp.h | 3 #define TRACE_SYSTEM xdp
12 #include <net/xdp.h>
32 const struct bpf_prog *xdp, u32 act),
34 TP_ARGS(dev, xdp, act),
43 __entry->prog_id = xdp->aux->id;
93 const struct bpf_prog *xdp,
98 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index),
124 __entry->prog_id = xdp->aux->id;
143 const struct bpf_prog *xdp,
147 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
[all …]
|
| /linux/drivers/net/ethernet/intel/idpf/ |
| H A D | xdp.c | 7 #include "xdp.h"
189 idpf_queue_set(XDP, xdpsq); in idpf_xdpsqs_get()
190 idpf_queue_set(XDP, xdpsq->complq); in idpf_xdpsqs_get()
219 if (!idpf_queue_has_clear(XDP, xdpsq)) in idpf_xdpsqs_put()
387 const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; in idpf_xdpmo_rx_hash() local
392 rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); in idpf_xdpmo_rx_hash()
394 idpf_xdp_get_qw0(&desc, xdp->desc); in idpf_xdpmo_rx_hash()
400 idpf_xdp_get_qw2(&desc, xdp->desc); in idpf_xdpmo_rx_hash()
408 const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; in idpf_xdpmo_rx_timestamp() local
414 rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); in idpf_xdpmo_rx_timestamp()
[all …]
|