1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Network memory 4 * 5 * Author: Mina Almasry <almasrymina@google.com> 6 */ 7 8 #ifndef _NET_NETMEM_H 9 #define _NET_NETMEM_H 10 11 #include <linux/dma-mapping.h> 12 #include <linux/mm.h> 13 #include <net/net_debug.h> 14 15 /* These fields in struct page are used by the page_pool and net stack: 16 * 17 * struct { 18 * unsigned long pp_magic; 19 * struct page_pool *pp; 20 * unsigned long _pp_mapping_pad; 21 * unsigned long dma_addr; 22 * atomic_long_t pp_ref_count; 23 * }; 24 * 25 * We mirror the page_pool fields here so the page_pool can access these 26 * fields without worrying whether the underlying fields belong to a 27 * page or netmem_desc. 28 * 29 * CAUTION: Do not update the fields in netmem_desc without also 30 * updating the anonymous aliasing union in struct net_iov. 31 */ 32 struct netmem_desc { 33 unsigned long _flags; 34 unsigned long pp_magic; 35 struct page_pool *pp; 36 unsigned long _pp_mapping_pad; 37 unsigned long dma_addr; 38 atomic_long_t pp_ref_count; 39 }; 40 41 #define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \ 42 static_assert(offsetof(struct page, pg) == \ 43 offsetof(struct netmem_desc, desc)) 44 NETMEM_DESC_ASSERT_OFFSET(flags, _flags); 45 NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic); 46 NETMEM_DESC_ASSERT_OFFSET(pp, pp); 47 NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 48 NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr); 49 NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 50 #undef NETMEM_DESC_ASSERT_OFFSET 51 52 /* 53 * Since struct netmem_desc uses the space in struct page, the size 54 * should be checked, until struct netmem_desc has its own instance from 55 * slab, to avoid conflicting with other members within struct page. 56 */ 57 static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount)); 58 59 /* net_iov */ 60 61 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); 62 63 /* We overload the LSB of the struct page pointer to indicate whether it's 64 * a page or net_iov. 65 */ 66 #define NET_IOV 0x01UL 67 68 enum net_iov_type { 69 NET_IOV_DMABUF, 70 NET_IOV_IOURING, 71 }; 72 73 /* A memory descriptor representing abstract networking I/O vectors, 74 * generally for non-pages memory that doesn't have its corresponding 75 * struct page and needs to be explicitly allocated through slab. 76 * 77 * net_iovs are allocated and used by networking code, and the size of 78 * the chunk is PAGE_SIZE. 79 * 80 * This memory can be any form of non-struct paged memory. Examples 81 * include imported dmabuf memory and imported io_uring memory. See 82 * net_iov_type for all the supported types. 83 * 84 * @pp_magic: pp field, similar to the one in struct page/struct 85 * netmem_desc. 86 * @pp: the pp this net_iov belongs to, if any. 87 * @dma_addr: the dma addrs of the net_iov. Needed for the network 88 * card to send/receive this net_iov. 89 * @pp_ref_count: the pp ref count of this net_iov, exactly the same 90 * usage as struct page/struct netmem_desc. 91 * @owner: the net_iov_area this net_iov belongs to, if any. 92 * @type: the type of the memory. Different types of net_iovs are 93 * supported. 94 */ 95 struct net_iov { 96 struct netmem_desc desc; 97 enum net_iov_type type; 98 struct net_iov_area *owner; 99 }; 100 101 struct net_iov_area { 102 /* Array of net_iovs for this area. */ 103 struct net_iov *niovs; 104 size_t num_niovs; 105 }; 106 107 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) 108 { 109 return niov->owner; 110 } 111 112 static inline unsigned int net_iov_idx(const struct net_iov *niov) 113 { 114 return niov - net_iov_owner(niov)->niovs; 115 } 116 117 /* Initialize a niov: stamp the owning area, the memory provider type. 118 */ 119 static inline void net_iov_init(struct net_iov *niov, 120 struct net_iov_area *owner, 121 enum net_iov_type type) 122 { 123 niov->owner = owner; 124 niov->type = type; 125 } 126 127 /* netmem */ 128 129 /** 130 * typedef netmem_ref - a nonexistent type marking a reference to generic 131 * network memory. 132 * 133 * A netmem_ref can be a struct page* or a struct net_iov* underneath. 134 * 135 * Use the supplied helpers to obtain the underlying memory pointer and fields. 136 */ 137 typedef unsigned long __bitwise netmem_ref; 138 139 static inline bool netmem_is_net_iov(const netmem_ref netmem) 140 { 141 return (__force unsigned long)netmem & NET_IOV; 142 } 143 144 /** 145 * __netmem_to_page - unsafely get pointer to the &page backing @netmem 146 * @netmem: netmem reference to convert 147 * 148 * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 149 * e.g. when it's a header buffer, performs faster and generates smaller 150 * object code (no check for the LSB, no WARN). When @netmem points to IOV, 151 * provokes undefined behaviour. 152 * 153 * Return: pointer to the &page (garbage if @netmem is not page-backed). 154 */ 155 static inline struct page *__netmem_to_page(netmem_ref netmem) 156 { 157 return (__force struct page *)netmem; 158 } 159 160 static inline struct page *netmem_to_page(netmem_ref netmem) 161 { 162 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 163 return NULL; 164 165 return __netmem_to_page(netmem); 166 } 167 168 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) 169 { 170 if (netmem_is_net_iov(netmem)) 171 return (struct net_iov *)((__force unsigned long)netmem & 172 ~NET_IOV); 173 174 DEBUG_NET_WARN_ON_ONCE(true); 175 return NULL; 176 } 177 178 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) 179 { 180 return (__force netmem_ref)((unsigned long)niov | NET_IOV); 181 } 182 183 #define page_to_netmem(p) (_Generic((p), \ 184 const struct page * : (__force const netmem_ref)(p), \ 185 struct page * : (__force netmem_ref)(p))) 186 187 /** 188 * virt_to_netmem - convert virtual memory pointer to a netmem reference 189 * @data: host memory pointer to convert 190 * 191 * Return: netmem reference to the &page backing this virtual address. 192 */ 193 static inline netmem_ref virt_to_netmem(const void *data) 194 { 195 return page_to_netmem(virt_to_page(data)); 196 } 197 198 static inline int netmem_ref_count(netmem_ref netmem) 199 { 200 /* The non-pp refcount of net_iov is always 1. On net_iov, we only 201 * support pp refcounting which uses the pp_ref_count field. 202 */ 203 if (netmem_is_net_iov(netmem)) 204 return 1; 205 206 return page_ref_count(netmem_to_page(netmem)); 207 } 208 209 static inline unsigned long netmem_pfn_trace(netmem_ref netmem) 210 { 211 if (netmem_is_net_iov(netmem)) 212 return 0; 213 214 return page_to_pfn(netmem_to_page(netmem)); 215 } 216 217 /* XXX: How to extract netmem_desc from page must be changed, once 218 * netmem_desc no longer overlays on page and will be allocated through 219 * slab. 220 */ 221 #define __pp_page_to_nmdesc(p) (_Generic((p), \ 222 const struct page * : (const struct netmem_desc *)(p), \ 223 struct page * : (struct netmem_desc *)(p))) 224 225 /* CAUTION: Check if the page is a pp page before calling this helper or 226 * know it's a pp page. 227 */ 228 #define pp_page_to_nmdesc(p) \ 229 ({ \ 230 DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p)); \ 231 __pp_page_to_nmdesc(p); \ 232 }) 233 234 /** 235 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing 236 * @netmem 237 * @netmem: netmem reference to convert 238 * 239 * Unsafe version that can be used only when @netmem is always backed by 240 * system memory, performs faster and generates smaller object code (no 241 * check for the LSB, no WARN). When @netmem points to IOV, provokes 242 * undefined behaviour. 243 * 244 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed 245 * by system memory). 246 */ 247 static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem) 248 { 249 return (__force struct netmem_desc *)netmem; 250 } 251 252 /* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for 253 * access to common fields. 254 * @netmem: netmem reference to get netmem_desc. 255 * 256 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same 257 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc. 258 * 259 * Return: the pointer to struct netmem_desc * regardless of its 260 * underlying type. 261 */ 262 static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem) 263 { 264 void *p = (void *)((__force unsigned long)netmem & ~NET_IOV); 265 266 if (netmem_is_net_iov(netmem)) 267 return &((struct net_iov *)p)->desc; 268 269 return __pp_page_to_nmdesc((struct page *)p); 270 } 271 272 /** 273 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 274 * @netmem: netmem reference to get the pointer from 275 * 276 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 277 * e.g. when it's a header buffer, performs faster and generates smaller 278 * object code (avoids clearing the LSB). When @netmem points to IOV, 279 * provokes invalid memory access. 280 * 281 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 282 */ 283 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 284 { 285 return __netmem_to_nmdesc(netmem)->pp; 286 } 287 288 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) 289 { 290 return netmem_to_nmdesc(netmem)->pp; 291 } 292 293 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) 294 { 295 return &netmem_to_nmdesc(netmem)->pp_ref_count; 296 } 297 298 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) 299 { 300 /* NUMA node preference only makes sense if we're allocating 301 * system memory. Memory providers (which give us net_iovs) 302 * choose for us. 303 */ 304 if (netmem_is_net_iov(netmem)) 305 return true; 306 307 return page_to_nid(netmem_to_page(netmem)) == pref_nid; 308 } 309 310 static inline netmem_ref netmem_compound_head(netmem_ref netmem) 311 { 312 /* niov are never compounded */ 313 if (netmem_is_net_iov(netmem)) 314 return netmem; 315 316 return page_to_netmem(compound_head(netmem_to_page(netmem))); 317 } 318 319 /** 320 * __netmem_address - unsafely get pointer to the memory backing @netmem 321 * @netmem: netmem reference to get the pointer for 322 * 323 * Unsafe version of netmem_address(). When @netmem is always page-backed, 324 * e.g. when it's a header buffer, performs faster and generates smaller 325 * object code (no check for the LSB). When @netmem points to IOV, provokes 326 * undefined behaviour. 327 * 328 * Return: pointer to the memory (garbage if @netmem is not page-backed). 329 */ 330 static inline void *__netmem_address(netmem_ref netmem) 331 { 332 return page_address(__netmem_to_page(netmem)); 333 } 334 335 static inline void *netmem_address(netmem_ref netmem) 336 { 337 if (netmem_is_net_iov(netmem)) 338 return NULL; 339 340 return __netmem_address(netmem); 341 } 342 343 /** 344 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 345 * @netmem: netmem reference to check 346 * 347 * Return: true if @netmem is page-backed and the page was allocated under 348 * memory pressure, false otherwise. 349 */ 350 static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 351 { 352 if (netmem_is_net_iov(netmem)) 353 return false; 354 355 return page_is_pfmemalloc(netmem_to_page(netmem)); 356 } 357 358 #define NETMEM_32BIT_ARCH_WITH_64BIT_DMA \ 359 (sizeof(dma_addr_t) > sizeof(unsigned long)) 360 361 static inline unsigned long netmem_dma_addr_encode(dma_addr_t addr) 362 { 363 if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA) 364 addr >>= PAGE_SHIFT; 365 366 return addr; 367 } 368 369 static inline dma_addr_t netmem_dma_addr_decode(unsigned long addr) 370 { 371 if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA) 372 return (dma_addr_t)addr << PAGE_SHIFT; 373 374 return addr; 375 } 376 377 static inline bool netmem_dma_addr_fits(dma_addr_t addr) 378 { 379 /* We assume page alignment to shave off bottom bits, 380 * if this "compression" doesn't work we need to drop. 381 */ 382 return addr == netmem_dma_addr_decode(netmem_dma_addr_encode(addr)); 383 } 384 385 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) 386 { 387 return netmem_to_nmdesc(netmem)->dma_addr; 388 } 389 390 #if defined(CONFIG_NET_DEVMEM) 391 static inline bool net_is_devmem_iov(const struct net_iov *niov) 392 { 393 return niov->type == NET_IOV_DMABUF; 394 } 395 #else 396 static inline bool net_is_devmem_iov(const struct net_iov *niov) 397 { 398 return false; 399 } 400 #endif 401 402 void __get_netmem(netmem_ref netmem); 403 void __put_netmem(netmem_ref netmem); 404 405 static __always_inline void get_netmem(netmem_ref netmem) 406 { 407 if (netmem_is_net_iov(netmem)) 408 __get_netmem(netmem); 409 else 410 get_page(netmem_to_page(netmem)); 411 } 412 413 static __always_inline void put_netmem(netmem_ref netmem) 414 { 415 if (netmem_is_net_iov(netmem)) 416 __put_netmem(netmem); 417 else 418 put_page(netmem_to_page(netmem)); 419 } 420 421 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ 422 do { \ 423 if (!netmem_is_net_iov(NETMEM)) \ 424 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ 425 else \ 426 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ 427 } while (0) 428 429 static inline void netmem_dma_unmap_page_attrs(struct device *dev, 430 dma_addr_t addr, size_t size, 431 enum dma_data_direction dir, 432 unsigned long attrs) 433 { 434 if (!addr) 435 return; 436 437 dma_unmap_page_attrs(dev, addr, size, dir, attrs); 438 } 439 440 #endif /* _NET_NETMEM_H */ 441