58 	      offsetof(struct xdp_buff_xsk, xdp.data));
589 void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
599 					    const struct libeth_xdp_buff *xdp)  in libeth_xdp_tx_queue_head()  argument
601 	const struct xdp_buff *base = &xdp->base;  in libeth_xdp_tx_queue_head()
604 		.data	= xdp->data,  in libeth_xdp_tx_queue_head()
605 		.len_fl	= (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST,  in libeth_xdp_tx_queue_head()
606 		.soff	= xdp_data_hard_end(base) - xdp->data,  in libeth_xdp_tx_queue_head()
638 			 struct libeth_xdp_buff *xdp,  in libeth_xdp_tx_queue_bulk()  argument
648 		libeth_xdp_return_buff_slow(xdp);  in libeth_xdp_tx_queue_bulk()
652 	if (!libeth_xdp_tx_queue_head(bq, xdp))  in libeth_xdp_tx_queue_bulk()
655 	sinfo = xdp_get_shared_info_from_buff(&xdp->base);  in libeth_xdp_tx_queue_bulk()
670 	xdp->data = NULL;  in libeth_xdp_tx_queue_bulk()
1248 #define libeth_xdp_return_buff(xdp)	__libeth_xdp_return_buff(xdp, true)  argument
1250 static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp,  in __libeth_xdp_return_buff()  argument
1253 	if (!xdp_buff_has_frags(&xdp->base))  in __libeth_xdp_return_buff()
1256 	libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base),  in __libeth_xdp_return_buff()
1260 	libeth_xdp_return_va(xdp->data, napi);  in __libeth_xdp_return_buff()
1261 	xdp->data = NULL;  in __libeth_xdp_return_buff()
1264 bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
1278 static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp,  in libeth_xdp_prepare_buff()  argument
1284 	xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset,  in libeth_xdp_prepare_buff()
1286 	xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq);  in libeth_xdp_prepare_buff()
1303 static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp,  in libeth_xdp_process_buff()  argument
1310 	if (xdp->data)  in libeth_xdp_process_buff()
1311 		return libeth_xdp_buff_add_frag(xdp, fqe, len);  in libeth_xdp_process_buff()
1313 	libeth_xdp_prepare_buff(xdp, fqe, len);  in libeth_xdp_process_buff()
1315 	prefetch(xdp->data);  in libeth_xdp_process_buff()
1330 			    const struct libeth_xdp_buff *xdp)  in libeth_xdp_buff_stats_frags()  argument
1334 	sinfo = xdp_get_shared_info_from_buff(&xdp->base);  in libeth_xdp_buff_stats_frags()
1340 			      struct libeth_xdp_buff *xdp,
1355 __libeth_xdp_run_prog(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_prog()  argument
1360 	act = bpf_prog_run_xdp(bq->prog, &xdp->base);  in __libeth_xdp_run_prog()
1368 		libeth_xdp_return_buff(xdp);  in __libeth_xdp_run_prog()
1374 		if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))  in __libeth_xdp_run_prog()
1377 		xdp->data = NULL;  in __libeth_xdp_run_prog()
1385 	return libeth_xdp_prog_exception(bq, xdp, act, 0);  in __libeth_xdp_run_prog()
1403 __libeth_xdp_run_flush(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_flush()  argument
1405 		       u32 (*run)(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_flush()  argument
1408 				     struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_flush()
1417 	act = run(xdp, bq);  in __libeth_xdp_run_flush()
1418 	if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk)))  in __libeth_xdp_run_flush()
1439 #define libeth_xdp_run_prog(xdp, bq, fl)				      \  argument
1440 	(__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog,		      \
1469 __libeth_xdp_run_pass(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1472 		      void (*prep)(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1474 		      bool (*run)(struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1477 				       const struct libeth_xdp_buff *xdp,  in __libeth_xdp_run_pass()  argument
1482 	rs->bytes += xdp->base.data_end - xdp->data;  in __libeth_xdp_run_pass()
1485 	if (xdp_buff_has_frags(&xdp->base))  in __libeth_xdp_run_pass()
1486 		libeth_xdp_buff_stats_frags(rs, xdp);  in __libeth_xdp_run_pass()
1489 		prep(xdp, md);  in __libeth_xdp_run_pass()
1494 	if (!run(xdp, bq))  in __libeth_xdp_run_pass()
1498 	skb = xdp_build_skb_from_buff(&xdp->base);  in __libeth_xdp_run_pass()
1500 		libeth_xdp_return_buff_slow(xdp);  in __libeth_xdp_run_pass()
1504 	xdp->data = NULL;  in __libeth_xdp_run_pass()
1506 	if (unlikely(!populate(skb, xdp, rs))) {  in __libeth_xdp_run_pass()
1514 static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp,  in libeth_xdp_prep_desc()  argument
1517 	xdp->desc = desc;  in libeth_xdp_prep_desc()
1533 #define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate)	      \  argument
1534 	__libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc,  \
1614 	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
1640 	bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
1643 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq)	      \
1645 	return libeth_##pfx##_run_prog(xdp, bq, flush);			      \
1655 	void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
1658 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq,	      \
1662 	return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run,	      \
1687 	__LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
1710 #define libeth_xdp_buff_to_rq(xdp, type, member)			      \  argument
1711 	container_of_const((xdp)->base.rxq, type, member)
1746 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);