43  * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
58 	      offsetof(struct xdp_buff_xsk, xdp.data));
114  * Return: number of XDP Tx queues the device needs to use.
317  * struct libeth_xdp_tx_frame - represents one XDP Tx element
324  * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
368  * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
369  * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
372  * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
376  * All XDP Tx operations except XSk xmit queue each frame to the bulk first
434  * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
437  * @flags: XDP Tx flags
494  * libeth_xdp_tx_xmit_bulk - main XDP Tx function
504  * Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for
589 void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
593  * @bq: XDP Tx bulk to queue the head frag to
594  * @xdp: XDP buffer with the head to queue
599 					    const struct libeth_xdp_buff *xdp)  in libeth_xdp_tx_queue_head()  argument
601 	const struct xdp_buff *base = &xdp->base;  in libeth_xdp_tx_queue_head()
604 		.data	= xdp->data,  in libeth_xdp_tx_queue_head()
605 		.len_fl	= (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST,  in libeth_xdp_tx_queue_head()
606 		.soff	= xdp_data_hard_end(base) - xdp->data,  in libeth_xdp_tx_queue_head()
619  * @bq: XDP Tx bulk to queue the frag to
630  * @bq: XDP Tx bulk to queue the frame to
631  * @xdp: XDP buffer to queue
638 			 struct libeth_xdp_buff *xdp,  in libeth_xdp_tx_queue_bulk()  argument
648 		libeth_xdp_return_buff_slow(xdp);  in libeth_xdp_tx_queue_bulk()
652 	if (!libeth_xdp_tx_queue_head(bq, xdp))  in libeth_xdp_tx_queue_bulk()
655 	sinfo = xdp_get_shared_info_from_buff(&xdp->base);  in libeth_xdp_tx_queue_bulk()
670 	xdp->data = NULL;  in libeth_xdp_tx_queue_bulk()
705  * @frm: XDP Tx frame from the bulk
710  * Return: XDP Tx descriptor with the synced DMA and other info to pass to
758  * __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk
760  * @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.)
817  * libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit
837  * @xf: pointer to the XDP frame
865  * libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head
866  * @bq: XDP Tx bulk to queue the head frag to
867  * @xdpf: XDP frame with the head to queue
900  * libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag
901  * @bq: XDP Tx bulk to queue the frag to
926  * libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame
927  * @bq: XDP Tx bulk to queue the frame to
928  * @xdpf: XDP frame to queue
977  * libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe
978  * @frm: XDP Tx frame from the bulk
983  * Return: XDP Tx descriptor with the mapped DMA and other info to pass to
1021  * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
1027  * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver
1039  * @bq: XDP Tx bulk to queue frames to
1040  * @frames: XDP frames passed by the stack
1043  * @flush_bulk: driver callback to flush an XDP xmit bulk
1044  * @finalize: driver callback to finalize sending XDP Tx frames on the queue
1092  * @fr: XDP frames to send
1096  * @fl: driver callback to flush an XDP xmit bulk
1129  * libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll
1131  * @prog: RCU pointer to the XDP program (can be %NULL)
1136  * Should be called on an onstack XDP Tx bulk before the NAPI polling loop.
1138  * assumes XDP is not enabled.
1172  * @src: XDP buffer stash placed on the queue
1194  * @dst: XDP buffer stash placed on the queue
1211  * libeth_xdp_return_stash - free an XDP buffer stash from a queue
1242  * @xdp: buffer to free
1246  * direct recycling. Zeroes @xdp->data to avoid UAFs.
1248 #define libeth_xdp_return_buff(xdp)	__libeth_xdp_return_buff(xdp, true)  argument
1250 static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp,  in __libeth_xdp_return_buff()  argument
1253 	if (!xdp_buff_has_frags(&xdp->base))  in __libeth_xdp_return_buff()
1256 	libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base),  in __libeth_xdp_return_buff()
1260 	libeth_xdp_return_va(xdp->data, napi);  in __libeth_xdp_return_buff()
1261 	xdp->data = NULL;  in __libeth_xdp_return_buff()
1264 bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
1270  * @xdp: XDP buffer to attach the head to
1274  * Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer
1279 static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp,  in libeth_xdp_prepare_buff()  argument
1286 	static_assert(offsetofend(typeof(xdp->base), flags) -  in libeth_xdp_prepare_buff()
1287 		      offsetof(typeof(xdp->base), frame_sz) ==  in libeth_xdp_prepare_buff()
1290 	*(u64 *)&xdp->base.frame_sz = fqe->truesize;  in libeth_xdp_prepare_buff()
1292 	xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq);  in libeth_xdp_prepare_buff()
1294 	xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset,  in libeth_xdp_prepare_buff()
1300  * @xdp: XDP buffer to attach the Rx buffer to
1304  * If the XDP buffer is empty, attaches the Rx buffer as head and initializes
1312 static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp,  in libeth_xdp_process_buff()  argument
1319 	if (xdp->data)  in libeth_xdp_process_buff()
1320 		return libeth_xdp_buff_add_frag(xdp, fqe, len);  in libeth_xdp_process_buff()
1322 	libeth_xdp_prepare_buff(xdp, fqe, len);  in libeth_xdp_process_buff()
1324 	prefetch(xdp->data);  in libeth_xdp_process_buff()
1330  * libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info
1332  * @xdp: buffer to account
1339 			    const struct libeth_xdp_buff *xdp)  in libeth_xdp_buff_stats_frags()  argument
1343 	sinfo = xdp_get_shared_info_from_buff(&xdp->base);  in libeth_xdp_buff_stats_frags()
1349 			      struct libeth_xdp_buff *xdp,
1353  * __libeth_xdp_run_prog - run XDP program on an XDP buffer
1354  * @xdp: XDP buffer to run the prog on
1357  * Internal inline abstraction to run XDP program. Handles ``XDP_DROP``
1359  * Reports an XDP prog exception on errors.
1364 __libeth_xdp_run_prog(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_prog()  argument
1369 	act = bpf_prog_run_xdp(bq->prog, &xdp->base);  in __libeth_xdp_run_prog()
1377 		libeth_xdp_return_buff(xdp);  in __libeth_xdp_run_prog()
1383 		if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))  in __libeth_xdp_run_prog()
1386 		xdp->data = NULL;  in __libeth_xdp_run_prog()
1394 	return libeth_xdp_prog_exception(bq, xdp, act, 0);  in __libeth_xdp_run_prog()
1398  * __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict
1399  * @xdp: XDP buffer to run the prog on
1401  * @run: internal callback for running XDP program
1405  * Internal inline abstraction to run XDP program and additionally handle
1406  * ``XDP_TX`` verdict. Used by both XDP and XSk, hence @run and @queue.
1412 __libeth_xdp_run_flush(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_flush()  argument
1414 		       u32 (*run)(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_flush()  argument
1417 				     struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_flush()
1426 	act = run(xdp, bq);  in __libeth_xdp_run_flush()
1427 	if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk)))  in __libeth_xdp_run_flush()
1436  * libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts
1437  * @xdp: XDP buffer to process
1438  * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
1441  * Run the attached XDP program and handle all possible verdicts. XSk has its
1448 #define libeth_xdp_run_prog(xdp, bq, fl)				      \  argument
1449 	(__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog,		      \
1454  * __libeth_xdp_run_pass - helper to run XDP program and handle the result
1455  * @xdp: XDP buffer to process
1456  * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1459  * @md: metadata that should be filled to the XDP buffer
1461  * @run: driver wrapper to run XDP program
1467  * 3) runs XDP program if present;
1474  * to the XDP buffer. If so, please use LIBETH_XDP_DEFINE_RUN{,_PASS}()
1478 __libeth_xdp_run_pass(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1481 		      void (*prep)(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1483 		      bool (*run)(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1486 				       const struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1491 	rs->bytes += xdp->base.data_end - xdp->data;  in __libeth_xdp_run_pass()
1494 	if (xdp_buff_has_frags(&xdp->base))  in __libeth_xdp_run_pass()
1495 		libeth_xdp_buff_stats_frags(rs, xdp);  in __libeth_xdp_run_pass()
1498 		prep(xdp, md);  in __libeth_xdp_run_pass()
1503 	if (!run(xdp, bq))  in __libeth_xdp_run_pass()
1507 	skb = xdp_build_skb_from_buff(&xdp->base);  in __libeth_xdp_run_pass()
1509 		libeth_xdp_return_buff_slow(xdp);  in __libeth_xdp_run_pass()
1513 	xdp->data = NULL;  in __libeth_xdp_run_pass()
1515 	if (unlikely(!populate(skb, xdp, rs))) {  in __libeth_xdp_run_pass()
1523 static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp,  in libeth_xdp_prep_desc()  argument
1526 	xdp->desc = desc;  in libeth_xdp_prep_desc()
1530  * libeth_xdp_run_pass - helper to run XDP program and handle the result
1531  * @xdp: XDP buffer to process
1532  * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1536  * @run: driver wrapper to run XDP program
1540  * means just writing the pointer to the HW descriptor as @xdp->desc.
1542 #define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate)	      \  argument
1543 	__libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc,  \
1553  * the XDP maps.
1595  *		driver_xdp_run(xdp, &bq, napi, &rs, desc);
1623 	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
1632  * LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function
1644  * LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function
1649 	bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
1652 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq)	      \
1654 	return libeth_##pfx##_run_prog(xdp, bq, flush);			      \
1660  * @run: driver callback to run XDP program (above)
1664 	void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
1667 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq,	      \
1671 	return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run,	      \
1678  * @run: name of the XDP prog run function to define
1683 	__LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XDP)
1696 	__LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
1709  * libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer
1710  * @xdp: &libeth_xdp_buff corresponding to the queue
1715  * HW descriptors. The helper can be used to quickly jump from an XDP buffer
1719 #define libeth_xdp_buff_to_rq(xdp, type, member)			      \  argument
1720 	container_of_const((xdp)->base.rxq, type, member)
1723  * libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata
1730  * the corresponding XDP RSS hash type. To be called at the end of
1734  * generate XDP RSS hash type for each packet type.
1755 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);
1766  * Complete an XDPSQE of any type of XDP frame. This includes DMA unmapping
1837  * libeth_xdp_set_features - set XDP features for netdev
1843  * of whether an XDP prog is attached to @dev.