1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * page_pool/helpers.h 4 * Author: Jesper Dangaard Brouer <netoptimizer@brouer.com> 5 * Copyright (C) 2016 Red Hat, Inc. 6 */ 7 8 /** 9 * DOC: page_pool allocator 10 * 11 * The page_pool allocator is optimized for the XDP mode that 12 * uses one frame per-page, but it can fallback on the 13 * regular page allocator APIs. 14 * 15 * Basic use involves replacing alloc_pages() calls with the 16 * page_pool_alloc_pages() call. Drivers should use 17 * page_pool_dev_alloc_pages() replacing dev_alloc_pages(). 18 * 19 * API keeps track of in-flight pages, in order to let API user know 20 * when it is safe to free a page_pool object. Thus, API users 21 * must call page_pool_put_page() to free the page, or attach 22 * the page to a page_pool-aware objects like skbs marked with 23 * skb_mark_for_recycle(). 24 * 25 * API user must call page_pool_put_page() once on a page, as it 26 * will either recycle the page, or in case of refcnt > 1, it will 27 * release the DMA mapping and in-flight state accounting. 28 */ 29 #ifndef _NET_PAGE_POOL_HELPERS_H 30 #define _NET_PAGE_POOL_HELPERS_H 31 32 #include <net/page_pool/types.h> 33 34 #ifdef CONFIG_PAGE_POOL_STATS 35 int page_pool_ethtool_stats_get_count(void); 36 u8 *page_pool_ethtool_stats_get_strings(u8 *data); 37 u64 *page_pool_ethtool_stats_get(u64 *data, void *stats); 38 39 /* 40 * Drivers that wish to harvest page pool stats and report them to users 41 * (perhaps via ethtool, debugfs, or another mechanism) can allocate a 42 * struct page_pool_stats call page_pool_get_stats to get stats for the specified pool. 43 */ 44 bool page_pool_get_stats(struct page_pool *pool, 45 struct page_pool_stats *stats); 46 #else 47 static inline int page_pool_ethtool_stats_get_count(void) 48 { 49 return 0; 50 } 51 52 static inline u8 *page_pool_ethtool_stats_get_strings(u8 *data) 53 { 54 return data; 55 } 56 57 static inline u64 *page_pool_ethtool_stats_get(u64 *data, void *stats) 58 { 59 return data; 60 } 61 #endif 62 63 /** 64 * page_pool_dev_alloc_pages() - allocate a page. 65 * @pool: pool from which to allocate 66 * 67 * Get a page from the page allocator or page_pool caches. 68 */ 69 static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) 70 { 71 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 72 73 return page_pool_alloc_pages(pool, gfp); 74 } 75 76 static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, 77 unsigned int *offset, 78 unsigned int size) 79 { 80 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 81 82 return page_pool_alloc_frag(pool, offset, size, gfp); 83 } 84 85 /** 86 * page_pool_get_dma_dir() - Retrieve the stored DMA direction. 87 * @pool: pool from which page was allocated 88 * 89 * Get the stored dma direction. A driver might decide to store this locally 90 * and avoid the extra cache line from page_pool to determine the direction. 91 */ 92 static 93 inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) 94 { 95 return pool->p.dma_dir; 96 } 97 98 /* pp_frag_count represents the number of writers who can update the page 99 * either by updating skb->data or via DMA mappings for the device. 100 * We can't rely on the page refcnt for that as we don't know who might be 101 * holding page references and we can't reliably destroy or sync DMA mappings 102 * of the fragments. 103 * 104 * When pp_frag_count reaches 0 we can either recycle the page if the page 105 * refcnt is 1 or return it back to the memory allocator and destroy any 106 * mappings we have. 107 */ 108 static inline void page_pool_fragment_page(struct page *page, long nr) 109 { 110 atomic_long_set(&page->pp_frag_count, nr); 111 } 112 113 static inline long page_pool_defrag_page(struct page *page, long nr) 114 { 115 long ret; 116 117 /* If nr == pp_frag_count then we have cleared all remaining 118 * references to the page. No need to actually overwrite it, instead 119 * we can leave this to be overwritten by the calling function. 120 * 121 * The main advantage to doing this is that an atomic_read is 122 * generally a much cheaper operation than an atomic update, 123 * especially when dealing with a page that may be partitioned 124 * into only 2 or 3 pieces. 125 */ 126 if (atomic_long_read(&page->pp_frag_count) == nr) 127 return 0; 128 129 ret = atomic_long_sub_return(nr, &page->pp_frag_count); 130 WARN_ON(ret < 0); 131 return ret; 132 } 133 134 static inline bool page_pool_is_last_frag(struct page_pool *pool, 135 struct page *page) 136 { 137 /* If fragments aren't enabled or count is 0 we were the last user */ 138 return !(pool->p.flags & PP_FLAG_PAGE_FRAG) || 139 (page_pool_defrag_page(page, 1) == 0); 140 } 141 142 /** 143 * page_pool_put_page() - release a reference to a page pool page 144 * @pool: pool from which page was allocated 145 * @page: page to release a reference on 146 * @dma_sync_size: how much of the page may have been touched by the device 147 * @allow_direct: released by the consumer, allow lockless caching 148 * 149 * The outcome of this depends on the page refcnt. If the driver bumps 150 * the refcnt > 1 this will unmap the page. If the page refcnt is 1 151 * the allocator owns the page and will try to recycle it in one of the pool 152 * caches. If PP_FLAG_DMA_SYNC_DEV is set, the page will be synced for_device 153 * using dma_sync_single_range_for_device(). 154 */ 155 static inline void page_pool_put_page(struct page_pool *pool, 156 struct page *page, 157 unsigned int dma_sync_size, 158 bool allow_direct) 159 { 160 /* When page_pool isn't compiled-in, net/core/xdp.c doesn't 161 * allow registering MEM_TYPE_PAGE_POOL, but shield linker. 162 */ 163 #ifdef CONFIG_PAGE_POOL 164 if (!page_pool_is_last_frag(pool, page)) 165 return; 166 167 page_pool_put_defragged_page(pool, page, dma_sync_size, allow_direct); 168 #endif 169 } 170 171 /** 172 * page_pool_put_full_page() - release a reference on a page pool page 173 * @pool: pool from which page was allocated 174 * @page: page to release a reference on 175 * @allow_direct: released by the consumer, allow lockless caching 176 * 177 * Similar to page_pool_put_page(), but will DMA sync the entire memory area 178 * as configured in &page_pool_params.max_len. 179 */ 180 static inline void page_pool_put_full_page(struct page_pool *pool, 181 struct page *page, bool allow_direct) 182 { 183 page_pool_put_page(pool, page, -1, allow_direct); 184 } 185 186 /** 187 * page_pool_recycle_direct() - release a reference on a page pool page 188 * @pool: pool from which page was allocated 189 * @page: page to release a reference on 190 * 191 * Similar to page_pool_put_full_page() but caller must guarantee safe context 192 * (e.g NAPI), since it will recycle the page directly into the pool fast cache. 193 */ 194 static inline void page_pool_recycle_direct(struct page_pool *pool, 195 struct page *page) 196 { 197 page_pool_put_full_page(pool, page, true); 198 } 199 200 #define PAGE_POOL_DMA_USE_PP_FRAG_COUNT \ 201 (sizeof(dma_addr_t) > sizeof(unsigned long)) 202 203 /** 204 * page_pool_get_dma_addr() - Retrieve the stored DMA address. 205 * @page: page allocated from a page pool 206 * 207 * Fetch the DMA address of the page. The page pool to which the page belongs 208 * must had been created with PP_FLAG_DMA_MAP. 209 */ 210 static inline dma_addr_t page_pool_get_dma_addr(struct page *page) 211 { 212 dma_addr_t ret = page->dma_addr; 213 214 if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT) 215 ret |= (dma_addr_t)page->dma_addr_upper << 16 << 16; 216 217 return ret; 218 } 219 220 static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr) 221 { 222 page->dma_addr = addr; 223 if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT) 224 page->dma_addr_upper = upper_32_bits(addr); 225 } 226 227 static inline bool page_pool_put(struct page_pool *pool) 228 { 229 return refcount_dec_and_test(&pool->user_cnt); 230 } 231 232 static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid) 233 { 234 if (unlikely(pool->p.nid != new_nid)) 235 page_pool_update_nid(pool, new_nid); 236 } 237 238 #endif /* _NET_PAGE_POOL_HELPERS_H */ 239