1 // SPDX-License-Identifier: GPL-2.0-only
8 #include <net/libeth/xdp.h>
22 	spin_lock_init(&lock->lock);  in __libeth_xdpsq_get()
23 	lock->share = true;  in __libeth_xdpsq_get()
29 		netdev_warn(dev, "XDPSQ sharing enabled, possible XDP Tx slowdown\n");  in __libeth_xdpsq_get()
41 	lock->share = false;  in __libeth_xdpsq_put()
45 void __acquires(&lock->lock)
48 	spin_lock(&lock->lock);  in __libeth_xdpsq_lock()
52 void __releases(&lock->lock)
55 	spin_unlock(&lock->lock);  in __libeth_xdpsq_unlock()
59 /* XDPSQ clean-up timers */
62  * libeth_xdpsq_init_timer - initialize an XDPSQ clean-up timer
68  * XDPSQ clean-up timers must be set up before using at the queue configuration
75 	timer->xdpsq = xdpsq;  in libeth_xdpsq_init_timer()
76 	timer->lock = lock;  in libeth_xdpsq_init_timer()
78 	INIT_DELAYED_WORK(&timer->dwork, poll);  in libeth_xdpsq_init_timer()
87 	if (frm->len_fl & LIBETH_XDP_TX_MULTI)  in libeth_xdp_tx_return_one()
88 		libeth_xdp_return_frags(frm->data + frm->soff, true);  in libeth_xdp_tx_return_one()
90 	libeth_xdp_return_va(frm->data, true);  in libeth_xdp_tx_return_one()
99 		if (!(frm->len_fl & LIBETH_XDP_TX_FIRST))  in libeth_xdp_tx_return_bulk()
114  * libeth_xdp_tx_exception - handle Tx exceptions of XDP frames
115  * @bq: XDP Tx frame bulk
120  * Reports XDP Tx exceptions, frees the frames that won't be sent or adjust
126 	const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent];  in libeth_xdp_tx_exception()
127 	u32 left = bq->count - sent;  in libeth_xdp_tx_exception()
130 		libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX);  in libeth_xdp_tx_exception()
133 		memmove(bq->bulk, pos, left * sizeof(*bq->bulk));  in libeth_xdp_tx_exception()
134 		bq->count = left;  in libeth_xdp_tx_exception()
144 		libeth_xdp_xmit_return_bulk(pos, left, bq->dev);  in libeth_xdp_tx_exception()
146 	bq->count = 0;  in libeth_xdp_tx_exception()
161 		if (frm->flags & LIBETH_XDP_TX_FIRST)  in libeth_xdp_xmit_return_bulk()
162 			dma = *libeth_xdp_xmit_frame_dma(frm->xdpf);  in libeth_xdp_xmit_return_bulk()
166 		dma_unmap_page(dev->dev.parent, dma, dma_unmap_len(frm, len),  in libeth_xdp_xmit_return_bulk()
170 		n += !!(frm->flags & LIBETH_XDP_TX_FIRST);  in libeth_xdp_xmit_return_bulk()
177 /* Rx polling path */
180  * libeth_xdp_load_stash - recreate an &xdp_buff from libeth_xdp buffer stash
192 	dst->data = src->data;  in libeth_xdp_load_stash()
193 	dst->base.data_end = src->data + src->len;  in libeth_xdp_load_stash()
194 	dst->base.data_meta = src->data;  in libeth_xdp_load_stash()
195 	dst->base.data_hard_start = src->data - src->headroom;  in libeth_xdp_load_stash()
197 	dst->base.frame_sz = src->frame_sz;  in libeth_xdp_load_stash()
198 	dst->base.flags = src->flags;  in libeth_xdp_load_stash()
203  * libeth_xdp_save_stash - convert &xdp_buff to a libeth_xdp buffer stash
205  * @src: source XDP buffer
208  * Use the fields from the passed XDP buffer to initialize the stash on the
215 	dst->data = src->data;  in libeth_xdp_save_stash()
216 	dst->headroom = src->data - src->base.data_hard_start;  in libeth_xdp_save_stash()
217 	dst->len = src->base.data_end - src->data;  in libeth_xdp_save_stash()
219 	dst->frame_sz = src->base.frame_sz;  in libeth_xdp_save_stash()
220 	dst->flags = src->base.flags;  in libeth_xdp_save_stash()
222 	WARN_ON_ONCE(dst->flags != src->base.flags);  in libeth_xdp_save_stash()
228 	LIBETH_XDP_ONSTACK_BUFF(xdp);  in __libeth_xdp_return_stash()
230 	libeth_xdp_load_stash(xdp, stash);  in __libeth_xdp_return_stash()
231 	libeth_xdp_return_buff_slow(xdp);  in __libeth_xdp_return_stash()
233 	stash->data = NULL;  in __libeth_xdp_return_stash()
238  * libeth_xdp_return_buff_slow - free &libeth_xdp_buff
239  * @xdp: buffer to free/return
242  * queue clean-ups etc., without unwanted inlining.
244 void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp)  in libeth_xdp_return_buff_slow()  argument
246 	__libeth_xdp_return_buff(xdp, false);  in libeth_xdp_return_buff_slow()
251  * libeth_xdp_buff_add_frag - add frag to XDP buffer
252  * @xdp: head XDP buffer
253  * @fqe: Rx buffer containing the frag
261 bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,  in libeth_xdp_buff_add_frag()  argument
265 	netmem_ref netmem = fqe->netmem;  in libeth_xdp_buff_add_frag()
267 	if (!xdp_buff_add_frag(&xdp->base, netmem,  in libeth_xdp_buff_add_frag()
268 			       fqe->offset + netmem_get_pp(netmem)->p.offset,  in libeth_xdp_buff_add_frag()
269 			       len, fqe->truesize))  in libeth_xdp_buff_add_frag()
276 	libeth_xdp_return_buff_slow(xdp);  in libeth_xdp_buff_add_frag()
283  * libeth_xdp_prog_exception - handle XDP prog exceptions
284  * @bq: XDP Tx bulk
285  * @xdp: buffer to process
286  * @act: original XDP prog verdict
291  * Reports invalid @act, XDP exception trace event and frees the buffer.
293  * Return: libeth_xdp XDP prog verdict.
296 				     struct libeth_xdp_buff *xdp,  in libeth_xdp_prog_exception()  argument
300 		bpf_warn_invalid_xdp_action(bq->dev, bq->prog, act);  in libeth_xdp_prog_exception()
302 	libeth_trace_xdp_exception(bq->dev, bq->prog, act);  in libeth_xdp_prog_exception()
304 	if (xdp->base.rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)  in libeth_xdp_prog_exception()
305 		return libeth_xsk_prog_exception(xdp, act, ret);  in libeth_xdp_prog_exception()
307 	libeth_xdp_return_buff_slow(xdp);  in libeth_xdp_prog_exception()
318 	if (unlikely(bq->count == XDP_BULK_QUEUE_SIZE))  in libeth_xdp_put_netmem_bulk()
321 	bq->q[bq->count++] = netmem;  in libeth_xdp_put_netmem_bulk()
325  * libeth_xdp_return_buff_bulk - free &xdp_buff as part of a bulk
327  * @bq: XDP frame bulk to store the buffer
340 	for (u32 i = 0; i < sinfo->nr_frags; i++)  in libeth_xdp_return_buff_bulk()
341 		libeth_xdp_put_netmem_bulk(skb_frag_netmem(&sinfo->frags[i]),  in libeth_xdp_return_buff_bulk()
352  * libeth_xdp_queue_threshold - calculate XDP queue clean/refill threshold
358  * always pow-2, closest to 1/4 of the queue length.
375 	return high - quarter <= quarter - low ? high : low;  in libeth_xdp_queue_threshold()
380  * __libeth_xdp_set_features - set XDP features for netdev
382  * @xmo: XDP metadata ops (Rx hints)
384  * @tmo: XSk Tx metadata ops (Tx hints)
388  * Use the non-underscored versions in drivers instead.
402 	dev->xdp_metadata_ops = xmo;  in __libeth_xdp_set_features()
406 	dev->xdp_zc_max_segs = zc_segs ? : 1;  in __libeth_xdp_set_features()
407 	dev->xsk_tx_metadata_ops = zc_segs ? tmo : NULL;  in __libeth_xdp_set_features()
412  * libeth_xdp_set_redirect - toggle the XDP redirect feature
414  * @enable: whether XDP is enabled
449 MODULE_DESCRIPTION("Common Ethernet library - XDP infra");