1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Network memory 4 * 5 * Author: Mina Almasry <almasrymina@google.com> 6 */ 7 8 #ifndef _NET_NETMEM_H 9 #define _NET_NETMEM_H 10 11 #include <linux/dma-mapping.h> 12 #include <linux/mm.h> 13 #include <net/net_debug.h> 14 15 /* net_iov */ 16 17 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); 18 19 /* We overload the LSB of the struct page pointer to indicate whether it's 20 * a page or net_iov. 21 */ 22 #define NET_IOV 0x01UL 23 24 enum net_iov_type { 25 NET_IOV_DMABUF, 26 NET_IOV_IOURING, 27 28 /* Force size to unsigned long to make the NET_IOV_ASSERTS below pass. 29 */ 30 NET_IOV_MAX = ULONG_MAX 31 }; 32 33 struct net_iov { 34 enum net_iov_type type; 35 unsigned long pp_magic; 36 struct page_pool *pp; 37 struct net_iov_area *owner; 38 unsigned long dma_addr; 39 atomic_long_t pp_ref_count; 40 }; 41 42 struct net_iov_area { 43 /* Array of net_iovs for this area. */ 44 struct net_iov *niovs; 45 size_t num_niovs; 46 47 /* Offset into the dma-buf where this chunk starts. */ 48 unsigned long base_virtual; 49 }; 50 51 /* These fields in struct page are used by the page_pool and net stack: 52 * 53 * struct { 54 * unsigned long pp_magic; 55 * struct page_pool *pp; 56 * unsigned long _pp_mapping_pad; 57 * unsigned long dma_addr; 58 * atomic_long_t pp_ref_count; 59 * }; 60 * 61 * We mirror the page_pool fields here so the page_pool can access these fields 62 * without worrying whether the underlying fields belong to a page or net_iov. 63 * 64 * The non-net stack fields of struct page are private to the mm stack and must 65 * never be mirrored to net_iov. 66 */ 67 #define NET_IOV_ASSERT_OFFSET(pg, iov) \ 68 static_assert(offsetof(struct page, pg) == \ 69 offsetof(struct net_iov, iov)) 70 NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic); 71 NET_IOV_ASSERT_OFFSET(pp, pp); 72 NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr); 73 NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 74 #undef NET_IOV_ASSERT_OFFSET 75 76 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) 77 { 78 return niov->owner; 79 } 80 81 static inline unsigned int net_iov_idx(const struct net_iov *niov) 82 { 83 return niov - net_iov_owner(niov)->niovs; 84 } 85 86 /* netmem */ 87 88 /** 89 * typedef netmem_ref - a nonexistent type marking a reference to generic 90 * network memory. 91 * 92 * A netmem_ref can be a struct page* or a struct net_iov* underneath. 93 * 94 * Use the supplied helpers to obtain the underlying memory pointer and fields. 95 */ 96 typedef unsigned long __bitwise netmem_ref; 97 98 static inline bool netmem_is_net_iov(const netmem_ref netmem) 99 { 100 return (__force unsigned long)netmem & NET_IOV; 101 } 102 103 /** 104 * __netmem_to_page - unsafely get pointer to the &page backing @netmem 105 * @netmem: netmem reference to convert 106 * 107 * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 108 * e.g. when it's a header buffer, performs faster and generates smaller 109 * object code (no check for the LSB, no WARN). When @netmem points to IOV, 110 * provokes undefined behaviour. 111 * 112 * Return: pointer to the &page (garbage if @netmem is not page-backed). 113 */ 114 static inline struct page *__netmem_to_page(netmem_ref netmem) 115 { 116 return (__force struct page *)netmem; 117 } 118 119 static inline struct page *netmem_to_page(netmem_ref netmem) 120 { 121 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 122 return NULL; 123 124 return __netmem_to_page(netmem); 125 } 126 127 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) 128 { 129 if (netmem_is_net_iov(netmem)) 130 return (struct net_iov *)((__force unsigned long)netmem & 131 ~NET_IOV); 132 133 DEBUG_NET_WARN_ON_ONCE(true); 134 return NULL; 135 } 136 137 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) 138 { 139 return (__force netmem_ref)((unsigned long)niov | NET_IOV); 140 } 141 142 #define page_to_netmem(p) (_Generic((p), \ 143 const struct page * : (__force const netmem_ref)(p), \ 144 struct page * : (__force netmem_ref)(p))) 145 146 /** 147 * virt_to_netmem - convert virtual memory pointer to a netmem reference 148 * @data: host memory pointer to convert 149 * 150 * Return: netmem reference to the &page backing this virtual address. 151 */ 152 static inline netmem_ref virt_to_netmem(const void *data) 153 { 154 return page_to_netmem(virt_to_page(data)); 155 } 156 157 static inline int netmem_ref_count(netmem_ref netmem) 158 { 159 /* The non-pp refcount of net_iov is always 1. On net_iov, we only 160 * support pp refcounting which uses the pp_ref_count field. 161 */ 162 if (netmem_is_net_iov(netmem)) 163 return 1; 164 165 return page_ref_count(netmem_to_page(netmem)); 166 } 167 168 static inline unsigned long netmem_pfn_trace(netmem_ref netmem) 169 { 170 if (netmem_is_net_iov(netmem)) 171 return 0; 172 173 return page_to_pfn(netmem_to_page(netmem)); 174 } 175 176 /* __netmem_clear_lsb - convert netmem_ref to struct net_iov * for access to 177 * common fields. 178 * @netmem: netmem reference to extract as net_iov. 179 * 180 * All the sub types of netmem_ref (page, net_iov) have the same pp, pp_magic, 181 * dma_addr, and pp_ref_count fields at the same offsets. Thus, we can access 182 * these fields without a type check to make sure that the underlying mem is 183 * net_iov or page. 184 * 185 * The resulting value of this function can only be used to access the fields 186 * that are NET_IOV_ASSERT_OFFSET'd. Accessing any other fields will result in 187 * undefined behavior. 188 * 189 * Return: the netmem_ref cast to net_iov* regardless of its underlying type. 190 */ 191 static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem) 192 { 193 return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV); 194 } 195 196 /** 197 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 198 * @netmem: netmem reference to get the pointer from 199 * 200 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 201 * e.g. when it's a header buffer, performs faster and generates smaller 202 * object code (avoids clearing the LSB). When @netmem points to IOV, 203 * provokes invalid memory access. 204 * 205 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 206 */ 207 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 208 { 209 return __netmem_to_page(netmem)->pp; 210 } 211 212 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) 213 { 214 return __netmem_clear_lsb(netmem)->pp; 215 } 216 217 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) 218 { 219 return &__netmem_clear_lsb(netmem)->pp_ref_count; 220 } 221 222 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) 223 { 224 /* NUMA node preference only makes sense if we're allocating 225 * system memory. Memory providers (which give us net_iovs) 226 * choose for us. 227 */ 228 if (netmem_is_net_iov(netmem)) 229 return true; 230 231 return page_to_nid(netmem_to_page(netmem)) == pref_nid; 232 } 233 234 static inline netmem_ref netmem_compound_head(netmem_ref netmem) 235 { 236 /* niov are never compounded */ 237 if (netmem_is_net_iov(netmem)) 238 return netmem; 239 240 return page_to_netmem(compound_head(netmem_to_page(netmem))); 241 } 242 243 /** 244 * __netmem_address - unsafely get pointer to the memory backing @netmem 245 * @netmem: netmem reference to get the pointer for 246 * 247 * Unsafe version of netmem_address(). When @netmem is always page-backed, 248 * e.g. when it's a header buffer, performs faster and generates smaller 249 * object code (no check for the LSB). When @netmem points to IOV, provokes 250 * undefined behaviour. 251 * 252 * Return: pointer to the memory (garbage if @netmem is not page-backed). 253 */ 254 static inline void *__netmem_address(netmem_ref netmem) 255 { 256 return page_address(__netmem_to_page(netmem)); 257 } 258 259 static inline void *netmem_address(netmem_ref netmem) 260 { 261 if (netmem_is_net_iov(netmem)) 262 return NULL; 263 264 return __netmem_address(netmem); 265 } 266 267 /** 268 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 269 * @netmem: netmem reference to check 270 * 271 * Return: true if @netmem is page-backed and the page was allocated under 272 * memory pressure, false otherwise. 273 */ 274 static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 275 { 276 if (netmem_is_net_iov(netmem)) 277 return false; 278 279 return page_is_pfmemalloc(netmem_to_page(netmem)); 280 } 281 282 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) 283 { 284 return __netmem_clear_lsb(netmem)->dma_addr; 285 } 286 287 void get_netmem(netmem_ref netmem); 288 void put_netmem(netmem_ref netmem); 289 290 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ 291 do { \ 292 if (!netmem_is_net_iov(NETMEM)) \ 293 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ 294 else \ 295 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ 296 } while (0) 297 298 static inline void netmem_dma_unmap_page_attrs(struct device *dev, 299 dma_addr_t addr, size_t size, 300 enum dma_data_direction dir, 301 unsigned long attrs) 302 { 303 if (!addr) 304 return; 305 306 dma_unmap_page_attrs(dev, addr, size, dir, attrs); 307 } 308 309 #endif /* _NET_NETMEM_H */ 310