30 	struct xsk_buff_pool *pool;  member
66 	/* For performance reasons, each buff pool has its own array of dma_pages
106 int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,
108 int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs,
110 int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs);
111 void xp_destroy(struct xsk_buff_pool *pool);
112 void xp_get_pool(struct xsk_buff_pool *pool);
113 bool xp_put_pool(struct xsk_buff_pool *pool);
114 void xp_clear_dev(struct xsk_buff_pool *pool);
115 void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
116 void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
121 static inline void xp_init_xskb_addr(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool,  in xp_init_xskb_addr()  argument
124 	xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom;  in xp_init_xskb_addr()
127 static inline void xp_init_xskb_dma(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool,  in xp_init_xskb_dma()  argument
132 	xskb->dma = xskb->frame_dma + pool->headroom + XDP_PACKET_HEADROOM;  in xp_init_xskb_dma()
136 void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq);
137 void xp_fill_cb(struct xsk_buff_pool *pool, struct xsk_cb_desc *desc);
138 int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
140 void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs);
141 struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool);
142 u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max);
143 bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count);
144 void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr);
145 dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr);
152 struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr);
166 	dma_sync_single_for_cpu(xskb->pool->dev, xskb->dma,  in xp_dma_sync_for_cpu()
167 				xskb->pool->frame_len,  in xp_dma_sync_for_cpu()
171 static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool,  in xp_dma_sync_for_device()  argument
174 	dma_sync_single_for_device(pool->dev, dma, size, DMA_BIDIRECTIONAL);  in xp_dma_sync_for_device()
184 static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool,  in xp_desc_crosses_non_contig_pg()  argument
192 	return pool->dma_pages &&  in xp_desc_crosses_non_contig_pg()
193 	       !(pool->dma_pages[addr >> PAGE_SHIFT] & XSK_NEXT_PG_CONTIG_MASK);  in xp_desc_crosses_non_contig_pg()
201 static inline u64 xp_aligned_extract_addr(struct xsk_buff_pool *pool, u64 addr)  in xp_aligned_extract_addr()  argument
203 	return addr & pool->chunk_mask;  in xp_aligned_extract_addr()
222 static inline u32 xp_aligned_extract_idx(struct xsk_buff_pool *pool, u64 addr)  in xp_aligned_extract_idx()  argument
224 	return xp_aligned_extract_addr(pool, addr) >> pool->chunk_shift;  in xp_aligned_extract_idx()
229 	if (xskb->pool->unaligned)  in xp_release()
230 		xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb;  in xp_release()
234 				struct xsk_buff_pool *pool)  in xp_get_handle()  argument
236 	u64 orig_addr = xskb->xdp.data - pool->addrs;  in xp_get_handle()
239 	if (!pool->unaligned)  in xp_get_handle()
243 	offset += pool->headroom;  in xp_get_handle()
248 static inline bool xp_tx_metadata_enabled(const struct xsk_buff_pool *pool)  in xp_tx_metadata_enabled()  argument
250 	return pool->tx_metadata_len > 0;  in xp_tx_metadata_enabled()