1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Network memory 4 * 5 * Author: Mina Almasry <almasrymina@google.com> 6 */ 7 8 #ifndef _NET_NETMEM_H 9 #define _NET_NETMEM_H 10 11 #include <linux/dma-mapping.h> 12 #include <linux/mm.h> 13 #include <net/net_debug.h> 14 15 /* These fields in struct page are used by the page_pool and net stack: 16 * 17 * struct { 18 * unsigned long pp_magic; 19 * struct page_pool *pp; 20 * unsigned long _pp_mapping_pad; 21 * unsigned long dma_addr; 22 * atomic_long_t pp_ref_count; 23 * }; 24 * 25 * We mirror the page_pool fields here so the page_pool can access these 26 * fields without worrying whether the underlying fields belong to a 27 * page or netmem_desc. 28 * 29 * CAUTION: Do not update the fields in netmem_desc without also 30 * updating the anonymous aliasing union in struct net_iov. 31 */ 32 struct netmem_desc { 33 unsigned long _flags; 34 unsigned long pp_magic; 35 struct page_pool *pp; 36 unsigned long _pp_mapping_pad; 37 unsigned long dma_addr; 38 atomic_long_t pp_ref_count; 39 }; 40 41 #define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \ 42 static_assert(offsetof(struct page, pg) == \ 43 offsetof(struct netmem_desc, desc)) 44 NETMEM_DESC_ASSERT_OFFSET(flags, _flags); 45 NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic); 46 NETMEM_DESC_ASSERT_OFFSET(pp, pp); 47 NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 48 NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr); 49 NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 50 #undef NETMEM_DESC_ASSERT_OFFSET 51 52 /* 53 * Since struct netmem_desc uses the space in struct page, the size 54 * should be checked, until struct netmem_desc has its own instance from 55 * slab, to avoid conflicting with other members within struct page. 56 */ 57 static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount)); 58 59 /* net_iov */ 60 61 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); 62 63 /* We overload the LSB of the struct page pointer to indicate whether it's 64 * a page or net_iov. 65 */ 66 #define NET_IOV 0x01UL 67 68 enum net_iov_type { 69 NET_IOV_DMABUF, 70 NET_IOV_IOURING, 71 }; 72 73 /* A memory descriptor representing abstract networking I/O vectors. 74 * 75 * net_iovs are allocated and used by networking code, and the size of 76 * the chunk is PAGE_SIZE. 77 * 78 * Examples include imported dmabuf memory and imported io_uring memory. See 79 * net_iov_type for all the supported types. While current net_iov types are 80 * unreadable by the CPU, net_iov has no inherent restrictions and may be 81 * CPU-readable or unreadable. New code must not assume net_iov implies 82 * unreadable memory (check readability via netmem_address() or 83 * skb_frags_readable() instead) and should, as much as possible, generalize 84 * existing limitations to match the design principles. 85 * 86 * @pp_magic: pp field, similar to the one in struct page/struct 87 * netmem_desc. 88 * @pp: the pp this net_iov belongs to, if any. 89 * @dma_addr: the dma addrs of the net_iov. Needed for the network 90 * card to send/receive this net_iov. 91 * @pp_ref_count: the pp ref count of this net_iov, exactly the same 92 * usage as struct page/struct netmem_desc. 93 * @owner: the net_iov_area this net_iov belongs to, if any. 94 * @type: the type of the memory. Different types of net_iovs are 95 * supported. 96 */ 97 struct net_iov { 98 struct netmem_desc desc; 99 enum net_iov_type type; 100 struct net_iov_area *owner; 101 }; 102 103 struct net_iov_area { 104 /* Array of net_iovs for this area. */ 105 struct net_iov *niovs; 106 size_t num_niovs; 107 }; 108 109 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) 110 { 111 return niov->owner; 112 } 113 114 static inline unsigned int net_iov_idx(const struct net_iov *niov) 115 { 116 return niov - net_iov_owner(niov)->niovs; 117 } 118 119 /* Initialize a niov: stamp the owning area, the memory provider type. 120 */ 121 static inline void net_iov_init(struct net_iov *niov, 122 struct net_iov_area *owner, 123 enum net_iov_type type) 124 { 125 niov->owner = owner; 126 niov->type = type; 127 } 128 129 /* netmem */ 130 131 /** 132 * typedef netmem_ref - a nonexistent type marking a reference to generic 133 * network memory. 134 * 135 * A netmem_ref can be a struct page* or a struct net_iov* underneath. 136 * Memory providers (or the default page_pool allocator) allocate struct 137 * net_iov or struct page, cast them to netmem_ref, and hand them to 138 * page_pool. 139 * 140 * The page_pool, drivers, and core networking stack should operate on 141 * netmem_ref rather than struct page or struct net_iov. Downcasting 142 * netmem_ref via netmem_to_page() or netmem_to_net_iov() in callers is 143 * not allowed unless a code path strictly requires a specific backing 144 * type (e.g., kmap_local_page()). In such cases, add a netmem helper here 145 * that handles both page and net_iov cases and returns an error if the 146 * underlying type cannot support the operation. 147 */ 148 typedef unsigned long __bitwise netmem_ref; 149 150 static inline bool netmem_is_net_iov(const netmem_ref netmem) 151 { 152 return (__force unsigned long)netmem & NET_IOV; 153 } 154 155 /** 156 * __netmem_to_page - unsafely get pointer to the &page backing @netmem 157 * @netmem: netmem reference to convert 158 * 159 * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 160 * e.g. when it's a header buffer, performs faster and generates smaller 161 * object code (no check for the LSB, no WARN). When @netmem points to IOV, 162 * provokes undefined behaviour. 163 * 164 * Return: pointer to the &page (garbage if @netmem is not page-backed). 165 */ 166 static inline struct page *__netmem_to_page(netmem_ref netmem) 167 { 168 return (__force struct page *)netmem; 169 } 170 171 static inline struct page *netmem_to_page(netmem_ref netmem) 172 { 173 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 174 return NULL; 175 176 return __netmem_to_page(netmem); 177 } 178 179 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) 180 { 181 if (netmem_is_net_iov(netmem)) 182 return (struct net_iov *)((__force unsigned long)netmem & 183 ~NET_IOV); 184 185 DEBUG_NET_WARN_ON_ONCE(true); 186 return NULL; 187 } 188 189 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) 190 { 191 return (__force netmem_ref)((unsigned long)niov | NET_IOV); 192 } 193 194 #define page_to_netmem(p) (_Generic((p), \ 195 const struct page * : (__force const netmem_ref)(p), \ 196 struct page * : (__force netmem_ref)(p))) 197 198 /** 199 * virt_to_netmem - convert virtual memory pointer to a netmem reference 200 * @data: host memory pointer to convert 201 * 202 * Return: netmem reference to the &page backing this virtual address. 203 */ 204 static inline netmem_ref virt_to_netmem(const void *data) 205 { 206 return page_to_netmem(virt_to_page(data)); 207 } 208 209 static inline int netmem_ref_count(netmem_ref netmem) 210 { 211 /* The non-pp refcount of net_iov is always 1. On net_iov, we only 212 * support pp refcounting which uses the pp_ref_count field. 213 */ 214 if (netmem_is_net_iov(netmem)) 215 return 1; 216 217 return page_ref_count(netmem_to_page(netmem)); 218 } 219 220 static inline unsigned long netmem_pfn_trace(netmem_ref netmem) 221 { 222 if (netmem_is_net_iov(netmem)) 223 return 0; 224 225 return page_to_pfn(netmem_to_page(netmem)); 226 } 227 228 /* XXX: How to extract netmem_desc from page must be changed, once 229 * netmem_desc no longer overlays on page and will be allocated through 230 * slab. 231 */ 232 #define __pp_page_to_nmdesc(p) (_Generic((p), \ 233 const struct page * : (const struct netmem_desc *)(p), \ 234 struct page * : (struct netmem_desc *)(p))) 235 236 /* CAUTION: Check if the page is a pp page before calling this helper or 237 * know it's a pp page. 238 */ 239 #define pp_page_to_nmdesc(p) \ 240 ({ \ 241 DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p)); \ 242 __pp_page_to_nmdesc(p); \ 243 }) 244 245 /** 246 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing 247 * @netmem 248 * @netmem: netmem reference to convert 249 * 250 * Unsafe version that can be used only when @netmem is always backed by 251 * system memory, performs faster and generates smaller object code (no 252 * check for the LSB, no WARN). When @netmem points to IOV, provokes 253 * undefined behaviour. 254 * 255 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed 256 * by system memory). 257 */ 258 static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem) 259 { 260 return (__force struct netmem_desc *)netmem; 261 } 262 263 /* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for 264 * access to common fields. 265 * @netmem: netmem reference to get netmem_desc. 266 * 267 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same 268 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc. 269 * 270 * Return: the pointer to struct netmem_desc * regardless of its 271 * underlying type. 272 */ 273 static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem) 274 { 275 void *p = (void *)((__force unsigned long)netmem & ~NET_IOV); 276 277 if (netmem_is_net_iov(netmem)) 278 return &((struct net_iov *)p)->desc; 279 280 return __pp_page_to_nmdesc((struct page *)p); 281 } 282 283 /** 284 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 285 * @netmem: netmem reference to get the pointer from 286 * 287 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 288 * e.g. when it's a header buffer, performs faster and generates smaller 289 * object code (avoids clearing the LSB). When @netmem points to IOV, 290 * provokes invalid memory access. 291 * 292 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 293 */ 294 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 295 { 296 return __netmem_to_nmdesc(netmem)->pp; 297 } 298 299 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) 300 { 301 return netmem_to_nmdesc(netmem)->pp; 302 } 303 304 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) 305 { 306 return &netmem_to_nmdesc(netmem)->pp_ref_count; 307 } 308 309 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) 310 { 311 /* NUMA node preference only applies to struct page; net_iovs are 312 * managed by their memory provider. 313 */ 314 if (netmem_is_net_iov(netmem)) 315 return true; 316 317 return page_to_nid(netmem_to_page(netmem)) == pref_nid; 318 } 319 320 static inline netmem_ref netmem_compound_head(netmem_ref netmem) 321 { 322 /* niov are never compounded */ 323 if (netmem_is_net_iov(netmem)) 324 return netmem; 325 326 return page_to_netmem(compound_head(netmem_to_page(netmem))); 327 } 328 329 /** 330 * __netmem_address - unsafely get pointer to the memory backing @netmem 331 * @netmem: netmem reference to get the pointer for 332 * 333 * Unsafe version of netmem_address(). When @netmem is always page-backed, 334 * e.g. when it's a header buffer, performs faster and generates smaller 335 * object code (no check for the LSB). When @netmem points to IOV, provokes 336 * undefined behaviour. 337 * 338 * Return: pointer to the memory (garbage if @netmem is not page-backed). 339 */ 340 static inline void *__netmem_address(netmem_ref netmem) 341 { 342 return page_address(__netmem_to_page(netmem)); 343 } 344 345 static inline void *netmem_address(netmem_ref netmem) 346 { 347 if (netmem_is_net_iov(netmem)) 348 return NULL; 349 350 return __netmem_address(netmem); 351 } 352 353 /** 354 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 355 * @netmem: netmem reference to check 356 * 357 * Return: true if @netmem is page-backed and the page was allocated under 358 * memory pressure, false otherwise. 359 */ 360 static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 361 { 362 if (netmem_is_net_iov(netmem)) 363 return false; 364 365 return page_is_pfmemalloc(netmem_to_page(netmem)); 366 } 367 368 #define NETMEM_32BIT_ARCH_WITH_64BIT_DMA \ 369 (sizeof(dma_addr_t) > sizeof(unsigned long)) 370 371 static inline unsigned long netmem_dma_addr_encode(dma_addr_t addr) 372 { 373 if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA) 374 addr >>= PAGE_SHIFT; 375 376 return addr; 377 } 378 379 static inline dma_addr_t netmem_dma_addr_decode(unsigned long addr) 380 { 381 if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA) 382 return (dma_addr_t)addr << PAGE_SHIFT; 383 384 return addr; 385 } 386 387 static inline bool netmem_dma_addr_fits(dma_addr_t addr) 388 { 389 /* We assume page alignment to shave off bottom bits, 390 * if this "compression" doesn't work we need to drop. 391 */ 392 return addr == netmem_dma_addr_decode(netmem_dma_addr_encode(addr)); 393 } 394 395 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) 396 { 397 return netmem_to_nmdesc(netmem)->dma_addr; 398 } 399 400 #if defined(CONFIG_NET_DEVMEM) 401 static inline bool net_is_devmem_iov(const struct net_iov *niov) 402 { 403 return niov->type == NET_IOV_DMABUF; 404 } 405 #else 406 static inline bool net_is_devmem_iov(const struct net_iov *niov) 407 { 408 return false; 409 } 410 #endif 411 412 void net_iov_inc_ref(struct net_iov *niov); 413 void net_iov_dec_ref(struct net_iov *niov); 414 415 /** 416 * netmem_inc_ref - acquire a non-page_pool reference on a netmem 417 * @netmem: netmem to reference 418 * 419 * Acquires a non-pp backing reference (page._refcount via get_page() or 420 * binding->ref via net_iov_inc_ref()) regardless of page_pool membership. 421 * Counterpart to netmem_dec_ref(). 422 */ 423 static __always_inline void netmem_inc_ref(netmem_ref netmem) 424 { 425 if (netmem_is_net_iov(netmem)) 426 net_iov_inc_ref(netmem_to_net_iov(netmem)); 427 else 428 get_page(netmem_to_page(netmem)); 429 } 430 431 /** 432 * netmem_dec_ref - release a non-page_pool reference on a netmem 433 * @netmem: netmem to unreference 434 * 435 * Drops a non-pp backing reference (page._refcount via put_page() or 436 * binding->ref via net_iov_dec_ref()) regardless of page_pool membership. 437 * Counterpart to netmem_inc_ref(). 438 */ 439 static __always_inline void netmem_dec_ref(netmem_ref netmem) 440 { 441 if (netmem_is_net_iov(netmem)) 442 net_iov_dec_ref(netmem_to_net_iov(netmem)); 443 else 444 put_page(netmem_to_page(netmem)); 445 } 446 447 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ 448 do { \ 449 if (!netmem_is_net_iov(NETMEM)) \ 450 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ 451 else \ 452 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ 453 } while (0) 454 455 static inline void netmem_dma_unmap_page_attrs(struct device *dev, 456 dma_addr_t addr, size_t size, 457 enum dma_data_direction dir, 458 unsigned long attrs) 459 { 460 if (!addr) 461 return; 462 463 dma_unmap_page_attrs(dev, addr, size, dir, attrs); 464 } 465 466 #endif /* _NET_NETMEM_H */ 467