| /linux/include/net/libeth/ |
| H A D | xsk.h | 25 struct libeth_xdp_buff *xdp) in libeth_xsk_tx_queue_head() argument
28 .xsk = xdp, in libeth_xsk_tx_queue_head()
29 __libeth_xdp_tx_len(xdp->base.data_end - xdp->data, in libeth_xsk_tx_queue_head()
33 if (likely(!xdp_buff_has_frags(&xdp->base))) in libeth_xsk_tx_queue_head()
65 struct libeth_xdp_buff *xdp, in libeth_xsk_tx_queue_bulk() argument
73 libeth_xsk_buff_free_slow(xdp); in libeth_xsk_tx_queue_bulk()
77 if (!libeth_xsk_tx_queue_head(bq, xdp)) in libeth_xsk_tx_queue_bulk()
80 for (const struct libeth_xdp_buff *head = xdp; ; ) { in libeth_xsk_tx_queue_bulk()
81 xdp = container_of(xsk_buff_get_frag(&head->base), in libeth_xsk_tx_queue_bulk()
82 typeof(*xdp), base); in libeth_xsk_tx_queue_bulk()
[all …]
|
| H A D | xdp.h | 58 offsetof(struct xdp_buff_xsk, xdp.data));
589 void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
599 const struct libeth_xdp_buff *xdp) in libeth_xdp_tx_queue_head() argument
601 const struct xdp_buff *base = &xdp->base; in libeth_xdp_tx_queue_head()
604 .data = xdp->data, in libeth_xdp_tx_queue_head()
605 .len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST, in libeth_xdp_tx_queue_head()
606 .soff = xdp_data_hard_end(base) - xdp->data, in libeth_xdp_tx_queue_head()
638 struct libeth_xdp_buff *xdp, in libeth_xdp_tx_queue_bulk() argument
648 libeth_xdp_return_buff_slow(xdp); in libeth_xdp_tx_queue_bulk()
652 if (!libeth_xdp_tx_queue_head(bq, xdp)) in libeth_xdp_tx_queue_bulk()
[all …]
|
| /linux/Documentation/bpf/ |
| H A D | redirect.rst | 25 :doc: xdp redirect
29 those that do, not all of them support non-linear frames. Non-linear xdp
45 sudo bpftrace -e 'tracepoint:xdp:* { @cnt[probe] = count(); }'
49 @cnt[tracepoint:xdp:mem_connect]: 18
50 @cnt[tracepoint:xdp:mem_disconnect]: 18
51 @cnt[tracepoint:xdp:xdp_exception]: 19605
52 @cnt[tracepoint:xdp:xdp_devmap_xmit]: 1393604
53 @cnt[tracepoint:xdp:xdp_redirect]: 22292200
56 The various xdp tracepoints can be found in ``source/include/trace/events/xdp.h``
64 'tracepoint:xdp:xdp_redirect*_err {@redir_errno[-args->err] = count();}
[all …]
|
| /linux/tools/testing/selftests/bpf/progs/ |
| H A D | xdp_features.c | 65 xdp_process_echo_packet(struct xdp_md *xdp, bool dut) in xdp_process_echo_packet() argument
67 void *data_end = (void *)(long)xdp->data_end; in xdp_process_echo_packet()
68 void *data = (void *)(long)xdp->data; in xdp_process_echo_packet()
135 xdp_update_stats(struct xdp_md *xdp, bool tx, bool dut) in xdp_update_stats() argument
139 if (xdp_process_echo_packet(xdp, tx)) in xdp_update_stats()
156 int xdp_tester_check_tx(struct xdp_md *xdp) in xdp_tester_check_tx() argument
158 xdp_update_stats(xdp, true, false); in xdp_tester_check_tx()
164 int xdp_tester_check_rx(struct xdp_md *xdp) in xdp_tester_check_rx() argument
166 xdp_update_stats(xdp, false, false); in xdp_tester_check_rx()
174 int xdp_do_pass(struct xdp_md *xdp) in xdp_do_pass() argument
[all …]
|
| H A D | test_xdp_do_redirect.c | 29 int xdp_redirect(struct xdp_md *xdp) in xdp_redirect() argument
31 __u32 *metadata = (void *)(long)xdp->data_meta; in xdp_redirect()
32 void *data_end = (void *)(long)xdp->data_end; in xdp_redirect()
33 void *data = (void *)(long)xdp->data; in xdp_redirect()
41 if (xdp->ingress_ifindex != (__u32)ifindex_in) in xdp_redirect()
55 if (bpf_xdp_adjust_meta(xdp, sizeof(__u64))) in xdp_redirect()
86 int xdp_count_pkts(struct xdp_md *xdp) in xdp_count_pkts() argument
88 void *data = (void *)(long)xdp->data; in xdp_count_pkts()
89 void *data_end = (void *)(long)xdp->data_end; in xdp_count_pkts()
102 int xdp_redirect_to_111(struct xdp_md *xdp) in xdp_redirect_to_111() argument
[all …]
|
| H A D | xsk_xdp_progs.c | 22 SEC("xdp.frags") int xsk_def_prog(struct xdp_md *xdp) in xsk_def_prog()
27 SEC("xdp.frags") int xsk_xdp_drop(struct xdp_md *xdp) in xsk_xdp_drop()
36 SEC("xdp.frags") int xsk_xdp_populate_metadata(struct xdp_md *xdp) in xsk_xdp_populate_metadata()
43 err = bpf_xdp_adjust_meta(xdp, -(int)sizeof(struct xdp_info)); in xsk_xdp_populate_metadata()
47 data = (void *)(long)xdp->data; in xsk_xdp_populate_metadata()
48 data_meta = (void *)(long)xdp->data_meta; in xsk_xdp_populate_metadata()
59 SEC("xdp") in in xsk_xdp_shared_umem()
19 xsk_def_prog(struct xdp_md * xdp) xsk_def_prog() argument
24 xsk_xdp_drop(struct xdp_md * xdp) xsk_xdp_drop() argument
33 xsk_xdp_populate_metadata(struct xdp_md * xdp) xsk_xdp_populate_metadata() argument
56 xsk_xdp_shared_umem(struct xdp_md * xdp) xsk_xdp_shared_umem() argument
[all...] |
| H A D | test_xdp.c | 79 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
81 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
82 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
182 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| H A D | test_xdp_loop.c | 75 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
77 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
78 void *data = (void *)(long)xdp->data; in handle_ipv4()
108 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
111 data = (void *)(long)xdp->data; in handle_ipv4()
112 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
148 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
150 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
151 void *data = (void *)(long)xdp->data; in handle_ipv6()
178 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| H A D | test_xdp_dynptr.c | 77 static __always_inline int handle_ipv4(struct xdp_md *xdp, struct bpf_dynptr *xdp_ptr) in handle_ipv4() argument
98 if (ethhdr_sz + iphdr_sz + tcphdr_sz > xdp->data_end - xdp->data) in handle_ipv4()
121 if (bpf_xdp_adjust_head(xdp, 0 - (int)iphdr_sz)) in handle_ipv4()
124 bpf_dynptr_from_xdp(xdp, 0, &new_xdp_ptr); in handle_ipv4()
159 static __always_inline int handle_ipv6(struct xdp_md *xdp, struct bpf_dynptr *xdp_ptr) in handle_ipv6() argument
177 if (ethhdr_sz + iphdr_sz + tcphdr_sz > xdp->data_end - xdp->data) in handle_ipv6()
200 if (bpf_xdp_adjust_head(xdp, 0 - (int)ipv6hdr_sz)) in handle_ipv6()
203 bpf_dynptr_from_xdp(xdp, 0, &new_xdp_ptr); in handle_ipv6()
231 int _xdp_tx_iptunnel(struct xdp_md *xdp) in _xdp_tx_iptunnel() argument
240 bpf_dynptr_from_xdp(xdp, 0, &ptr); in _xdp_tx_iptunnel()
[all …]
|
| H A D | test_xdp_bpf2bpf.c | 45 int BPF_PROG(trace_on_entry, struct xdp_buff *xdp) in BPF_PROG() argument
49 meta.ifindex = xdp->rxq->dev->ifindex; in BPF_PROG()
50 meta.pkt_len = bpf_xdp_get_buff_len((struct xdp_md *)xdp); in BPF_PROG()
51 bpf_xdp_output(xdp, &perf_buf_map, in BPF_PROG()
56 test_result_fentry = xdp->rxq->dev->ifindex; in BPF_PROG()
62 int BPF_PROG(trace_on_exit, struct xdp_buff *xdp, int ret) in BPF_PROG() argument
|
| H A D | test_parse_tcp_hdr_opt.c | 36 static int parse_hdr_opt(const struct xdp_md *xdp, struct hdr_opt_state *state) in parse_hdr_opt() argument
38 const void *data = (void *)(long)xdp->data; in parse_hdr_opt()
39 const void *data_end = (void *)(long)xdp->data_end; in parse_hdr_opt()
82 int xdp_ingress_v6(struct xdp_md *xdp) in xdp_ingress_v6() argument
84 const void *data = (void *)(long)xdp->data; in xdp_ingress_v6()
85 const void *data_end = (void *)(long)xdp->data_end; in xdp_ingress_v6()
106 err = parse_hdr_opt(xdp, &opt_state); in xdp_ingress_v6()
|
| H A D | test_xdp_adjust_tail_shrink.c | 13 int _xdp_adjust_tail_shrink(struct xdp_md *xdp) in _xdp_adjust_tail_shrink() argument
15 __u8 *data_end = (void *)(long)xdp->data_end; in _xdp_adjust_tail_shrink()
16 __u8 *data = (void *)(long)xdp->data; in _xdp_adjust_tail_shrink()
19 switch (bpf_xdp_get_buff_len(xdp)) { in _xdp_adjust_tail_shrink()
47 if (bpf_xdp_adjust_tail(xdp, 0 - offset)) in _xdp_adjust_tail_shrink()
|
| H A D | test_xdp_update_frags.c | 14 int xdp_adjust_frags(struct xdp_md *xdp) in xdp_adjust_frags() argument
16 __u8 *data_end = (void *)(long)xdp->data_end; in xdp_adjust_frags()
17 __u8 *data = (void *)(long)xdp->data; in xdp_adjust_frags()
26 err = bpf_xdp_load_bytes(xdp, offset, val, sizeof(val)); in xdp_adjust_frags()
35 err = bpf_xdp_store_bytes(xdp, offset, val, sizeof(val)); in xdp_adjust_frags()
|
| /linux/drivers/net/ethernet/intel/igb/ |
| H A D | igb_xsk.c | 176 static u16 igb_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, in igb_fill_rx_descs() argument
187 buffs = xsk_buff_alloc_batch(pool, xdp, count); in igb_fill_rx_descs()
189 dma = xsk_buff_xdp_get_dma(*xdp); in igb_fill_rx_descs()
194 xdp++; in igb_fill_rx_descs()
207 struct xdp_buff **xdp; in igb_alloc_rx_buffers_zc() local
210 xdp = &rx_ring->rx_buffer_info_zc[ntu]; in igb_alloc_rx_buffers_zc()
213 nb_buffs_extra = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, in igb_alloc_rx_buffers_zc()
220 xdp = rx_ring->rx_buffer_info_zc; in igb_alloc_rx_buffers_zc()
225 nb_buffs = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, count); in igb_alloc_rx_buffers_zc()
256 struct xdp_buff *xdp = rx_ring->rx_buffer_info_zc[ntc]; in igb_clean_rx_ring_zc() local
[all …]
|
| /linux/drivers/net/ethernet/intel/libeth/ |
| H A D | xsk.c | 36 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp) in libeth_xsk_buff_free_slow() argument
38 xsk_buff_free(&xdp->base); in libeth_xsk_buff_free_slow()
54 struct libeth_xdp_buff *xdp) in libeth_xsk_buff_add_frag() argument
56 if (!xsk_buff_add_frag(&head->base, &xdp->base)) in libeth_xsk_buff_add_frag()
62 libeth_xsk_buff_free_slow(xdp); in libeth_xsk_buff_add_frag()
78 const struct libeth_xdp_buff *xdp) in libeth_xsk_buff_stats_frags() argument
80 libeth_xdp_buff_stats_frags(rs, xdp); in libeth_xsk_buff_stats_frags()
98 u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp, in __libeth_xsk_run_prog_slow() argument
104 xsk_buff_free(&xdp->base); in __libeth_xsk_run_prog_slow()
115 return libeth_xdp_prog_exception(bq, xdp, act, ret); in __libeth_xsk_run_prog_slow()
[all …]
|
| /linux/drivers/net/ethernet/intel/ixgbe/ |
| H A D | ixgbe_xsk.c | 99 struct xdp_buff *xdp) in ixgbe_run_xdp_zc() argument
108 act = bpf_prog_run_xdp(xdp_prog, xdp); in ixgbe_run_xdp_zc()
111 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); in ixgbe_run_xdp_zc()
125 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
168 bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); in ixgbe_alloc_rx_buffers_zc()
169 if (!bi->xdp) { in ixgbe_alloc_rx_buffers_zc()
174 dma = xsk_buff_xdp_get_dma(bi->xdp); in ixgbe_alloc_rx_buffers_zc()
214 const struct xdp_buff *xdp) in ixgbe_construct_skb_zc() argument
216 unsigned int totalsize = xdp->data_end - xdp->data_meta; in ixgbe_construct_skb_zc()
217 unsigned int metasize = xdp->data - xdp->data_meta; in ixgbe_construct_skb_zc()
[all …]
|
| /linux/samples/bpf/ |
| H A D | xdp_tx_iptunnel_kern.c | 77 static __always_inline int handle_ipv4(struct xdp_md *xdp) in handle_ipv4() argument
79 void *data_end = (void *)(long)xdp->data_end; in handle_ipv4()
80 void *data = (void *)(long)xdp->data; in handle_ipv4()
112 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) in handle_ipv4()
115 data = (void *)(long)xdp->data; in handle_ipv4()
116 data_end = (void *)(long)xdp->data_end; in handle_ipv4()
152 static __always_inline int handle_ipv6(struct xdp_md *xdp) in handle_ipv6() argument
154 void *data_end = (void *)(long)xdp->data_end; in handle_ipv6()
155 void *data = (void *)(long)xdp->data; in handle_ipv6()
184 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) in handle_ipv6()
[all …]
|
| /linux/drivers/net/ethernet/microchip/lan966x/ |
| H A D | lan966x_xdp.c | 9 static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp_setup() argument
18 NL_SET_ERR_MSG_MOD(xdp->extack, in lan966x_xdp_setup()
24 old_prog = xchg(&port->xdp_prog, xdp->prog); in lan966x_xdp_setup()
43 int lan966x_xdp(struct net_device *dev, struct netdev_bpf *xdp) in lan966x_xdp() argument
45 switch (xdp->command) { in lan966x_xdp()
47 return lan966x_xdp_setup(dev, xdp); in lan966x_xdp()
79 struct xdp_buff xdp; in lan966x_xdp_run() local
82 xdp_init_buff(&xdp, PAGE_SIZE << lan966x->rx.page_order, in lan966x_xdp_run()
84 xdp_prepare_buff(&xdp, page_address(page), in lan966x_xdp_run()
87 act = bpf_prog_run_xdp(xdp_prog, &xdp); in lan966x_xdp_run()
[all …]
|
| /linux/drivers/net/vmxnet3/ |
| H A D | vmxnet3_xdp.c | 261 vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct xdp_buff *xdp, in vmxnet3_run_xdp() argument
270 act = bpf_prog_run_xdp(prog, xdp); in vmxnet3_run_xdp()
271 page = virt_to_page(xdp->data_hard_start); in vmxnet3_run_xdp()
277 err = xdp_do_redirect(rq->adapter->netdev, xdp, prog); in vmxnet3_run_xdp()
286 xdpf = xdp_convert_buff_to_frame(xdp); in vmxnet3_run_xdp()
314 const struct xdp_buff *xdp) in vmxnet3_build_skb() argument
326 skb_reserve(skb, xdp->data - xdp->data_hard_start); in vmxnet3_build_skb()
327 skb_put(skb, xdp->data_end - xdp->data); in vmxnet3_build_skb()
341 struct xdp_buff xdp; in vmxnet3_process_xdp_small() local
351 xdp_init_buff(&xdp, PAGE_SIZE, &rq->xdp_rxq); in vmxnet3_process_xdp_small()
[all …]
|
| /linux/drivers/net/ethernet/netronome/nfp/ |
| H A D | nfp_net_xsk.c | 18 struct xdp_buff *xdp) in nfp_net_xsk_rx_bufs_stash() argument
27 rx_ring->xsk_rxbufs[idx].xdp = xdp; in nfp_net_xsk_rx_bufs_stash()
29 xsk_buff_xdp_get_frame_dma(xdp) + headroom; in nfp_net_xsk_rx_bufs_stash()
35 rxbuf->xdp = NULL; in nfp_net_xsk_rx_unstash()
40 if (rxbuf->xdp) in nfp_net_xsk_rx_free()
41 xsk_buff_free(rxbuf->xdp); in nfp_net_xsk_rx_free()
62 struct xdp_buff *xdp; in nfp_net_xsk_rx_ring_fill_freelist() local
67 xdp = xsk_buff_alloc(pool); in nfp_net_xsk_rx_ring_fill_freelist()
68 if (!xdp) in nfp_net_xsk_rx_ring_fill_freelist()
71 nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp); in nfp_net_xsk_rx_ring_fill_freelist()
|
| /linux/drivers/net/hyperv/ |
| H A D | netvsc_bpf.c | 25 struct xdp_buff *xdp) in netvsc_run_xdp() argument
35 xdp->data_hard_start = NULL; in netvsc_run_xdp()
56 xdp_init_buff(xdp, PAGE_SIZE, &nvchan->xdp_rxq); in netvsc_run_xdp()
57 xdp_prepare_buff(xdp, page_address(page), NETVSC_XDP_HDRM, len, false); in netvsc_run_xdp()
59 memcpy(xdp->data, data, len); in netvsc_run_xdp()
61 act = bpf_prog_run_xdp(prog, xdp); in netvsc_run_xdp()
73 if (!xdp_do_redirect(ndev, xdp, prog)) { in netvsc_run_xdp()
107 xdp->data_hard_start = NULL; in netvsc_run_xdp()
167 struct netdev_bpf xdp; in netvsc_vf_setxdp() local
178 memset(&xdp, 0, sizeof(xdp)); in netvsc_vf_setxdp()
[all …]
|
| /linux/include/trace/events/ |
| H A D | xdp.h | 3 #define TRACE_SYSTEM xdp
32 const struct bpf_prog *xdp, u32 act),
34 TP_ARGS(dev, xdp, act),
43 __entry->prog_id = xdp->aux->id;
93 const struct bpf_prog *xdp,
98 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index),
124 __entry->prog_id = xdp->aux->id;
143 const struct bpf_prog *xdp,
147 TP_ARGS(dev, xdp, tgt, err, map_type, map_id, index)
152 const struct bpf_prog *xdp,
[all …]
|
| /linux/net/bpf/ |
| H A D | test_run.c | 125 struct xdp_test_data *xdp = arg; in xdp_test_run_init_page() local
130 orig_ctx = xdp->orig_ctx; in xdp_test_run_init_page()
140 xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq); in xdp_test_run_init_page()
150 static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx) in xdp_test_run_setup() argument
157 .pool_size = xdp->batch_size, in xdp_test_run_setup()
160 .init_arg = xdp, in xdp_test_run_setup()
163 xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
164 if (!xdp->frames) in xdp_test_run_setup()
167 xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL); in xdp_test_run_setup()
168 if (!xdp->skbs) in xdp_test_run_setup()
[all …]
|
| /linux/drivers/net/ethernet/broadcom/bnxt/ |
| H A D | bnxt_xdp.h | 18 struct xdp_buff *xdp);
21 struct xdp_buff *xdp, struct page *page, u8 **data_ptr,
23 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp);
31 struct xdp_buff *xdp);
33 struct xdp_buff *xdp);
36 struct xdp_buff *xdp);
|
| /linux/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/ |
| H A D | rx.c | 11 static struct mlx5e_xdp_buff *xsk_buff_to_mxbuf(struct xdp_buff *xdp) in xsk_buff_to_mxbuf() argument
16 return (struct mlx5e_xdp_buff *)xdp; in xsk_buff_to_mxbuf()
225 static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_buff *xdp) in mlx5e_xsk_construct_skb() argument
227 u32 totallen = xdp->data_end - xdp->data_meta; in mlx5e_xsk_construct_skb()
228 u32 metalen = xdp->data - xdp->data_meta; in mlx5e_xsk_construct_skb()
237 skb_put_data(skb, xdp->data_meta, totallen); in mlx5e_xsk_construct_skb()
272 xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
273 xsk_buff_dma_sync_for_cpu(&mxbuf->xdp); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
274 net_prefetch(mxbuf->xdp.data); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
301 return mlx5e_xsk_construct_skb(rq, &mxbuf->xdp); in mlx5e_xsk_skb_from_cqe_mpwrq_linear()
[all …]
|