1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Network memory 4 * 5 * Author: Mina Almasry <almasrymina@google.com> 6 */ 7 8 #ifndef _NET_NETMEM_H 9 #define _NET_NETMEM_H 10 11 #include <linux/dma-mapping.h> 12 #include <linux/mm.h> 13 #include <net/net_debug.h> 14 15 /* These fields in struct page are used by the page_pool and net stack: 16 * 17 * struct { 18 * unsigned long pp_magic; 19 * struct page_pool *pp; 20 * unsigned long _pp_mapping_pad; 21 * unsigned long dma_addr; 22 * atomic_long_t pp_ref_count; 23 * }; 24 * 25 * We mirror the page_pool fields here so the page_pool can access these 26 * fields without worrying whether the underlying fields belong to a 27 * page or netmem_desc. 28 * 29 * CAUTION: Do not update the fields in netmem_desc without also 30 * updating the anonymous aliasing union in struct net_iov. 31 */ 32 struct netmem_desc { 33 unsigned long _flags; 34 unsigned long pp_magic; 35 struct page_pool *pp; 36 unsigned long _pp_mapping_pad; 37 unsigned long dma_addr; 38 atomic_long_t pp_ref_count; 39 }; 40 41 #define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \ 42 static_assert(offsetof(struct page, pg) == \ 43 offsetof(struct netmem_desc, desc)) 44 NETMEM_DESC_ASSERT_OFFSET(flags, _flags); 45 NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic); 46 NETMEM_DESC_ASSERT_OFFSET(pp, pp); 47 NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 48 NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr); 49 NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 50 #undef NETMEM_DESC_ASSERT_OFFSET 51 52 /* 53 * Since struct netmem_desc uses the space in struct page, the size 54 * should be checked, until struct netmem_desc has its own instance from 55 * slab, to avoid conflicting with other members within struct page. 56 */ 57 static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount)); 58 59 /* net_iov */ 60 61 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); 62 63 /* We overload the LSB of the struct page pointer to indicate whether it's 64 * a page or net_iov. 65 */ 66 #define NET_IOV 0x01UL 67 68 enum net_iov_type { 69 NET_IOV_DMABUF, 70 NET_IOV_IOURING, 71 }; 72 73 /* A memory descriptor representing abstract networking I/O vectors, 74 * generally for non-pages memory that doesn't have its corresponding 75 * struct page and needs to be explicitly allocated through slab. 76 * 77 * net_iovs are allocated and used by networking code, and the size of 78 * the chunk is PAGE_SIZE. 79 * 80 * This memory can be any form of non-struct paged memory. Examples 81 * include imported dmabuf memory and imported io_uring memory. See 82 * net_iov_type for all the supported types. 83 * 84 * @pp_magic: pp field, similar to the one in struct page/struct 85 * netmem_desc. 86 * @pp: the pp this net_iov belongs to, if any. 87 * @dma_addr: the dma addrs of the net_iov. Needed for the network 88 * card to send/receive this net_iov. 89 * @pp_ref_count: the pp ref count of this net_iov, exactly the same 90 * usage as struct page/struct netmem_desc. 91 * @owner: the net_iov_area this net_iov belongs to, if any. 92 * @type: the type of the memory. Different types of net_iovs are 93 * supported. 94 */ 95 struct net_iov { 96 union { 97 struct netmem_desc desc; 98 99 /* XXX: The following part should be removed once all 100 * the references to them are converted so as to be 101 * accessed via netmem_desc e.g. niov->desc.pp instead 102 * of niov->pp. 103 */ 104 struct { 105 unsigned long _flags; 106 unsigned long pp_magic; 107 struct page_pool *pp; 108 unsigned long _pp_mapping_pad; 109 unsigned long dma_addr; 110 atomic_long_t pp_ref_count; 111 }; 112 }; 113 114 unsigned int page_type; 115 enum net_iov_type type; 116 struct net_iov_area *owner; 117 }; 118 119 /* Make sure 'the offset of page_type in struct page == the offset of 120 * type in struct net_iov'. 121 */ 122 #define NET_IOV_ASSERT_OFFSET(pg, iov) \ 123 static_assert(offsetof(struct page, pg) == \ 124 offsetof(struct net_iov, iov)) 125 NET_IOV_ASSERT_OFFSET(page_type, page_type); 126 #undef NET_IOV_ASSERT_OFFSET 127 128 struct net_iov_area { 129 /* Array of net_iovs for this area. */ 130 struct net_iov *niovs; 131 size_t num_niovs; 132 133 /* Offset into the dma-buf where this chunk starts. */ 134 unsigned long base_virtual; 135 }; 136 137 /* net_iov is union'ed with struct netmem_desc mirroring struct page, so 138 * the page_pool can access these fields without worrying whether the 139 * underlying fields are accessed via netmem_desc or directly via 140 * net_iov, until all the references to them are converted so as to be 141 * accessed via netmem_desc e.g. niov->desc.pp instead of niov->pp. 142 * 143 * The non-net stack fields of struct page are private to the mm stack 144 * and must never be mirrored to net_iov. 145 */ 146 #define NET_IOV_ASSERT_OFFSET(desc, iov) \ 147 static_assert(offsetof(struct netmem_desc, desc) == \ 148 offsetof(struct net_iov, iov)) 149 NET_IOV_ASSERT_OFFSET(_flags, _flags); 150 NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic); 151 NET_IOV_ASSERT_OFFSET(pp, pp); 152 NET_IOV_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 153 NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr); 154 NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 155 #undef NET_IOV_ASSERT_OFFSET 156 157 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) 158 { 159 return niov->owner; 160 } 161 162 static inline unsigned int net_iov_idx(const struct net_iov *niov) 163 { 164 return niov - net_iov_owner(niov)->niovs; 165 } 166 167 /* netmem */ 168 169 /** 170 * typedef netmem_ref - a nonexistent type marking a reference to generic 171 * network memory. 172 * 173 * A netmem_ref can be a struct page* or a struct net_iov* underneath. 174 * 175 * Use the supplied helpers to obtain the underlying memory pointer and fields. 176 */ 177 typedef unsigned long __bitwise netmem_ref; 178 179 static inline bool netmem_is_net_iov(const netmem_ref netmem) 180 { 181 return (__force unsigned long)netmem & NET_IOV; 182 } 183 184 /** 185 * __netmem_to_page - unsafely get pointer to the &page backing @netmem 186 * @netmem: netmem reference to convert 187 * 188 * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 189 * e.g. when it's a header buffer, performs faster and generates smaller 190 * object code (no check for the LSB, no WARN). When @netmem points to IOV, 191 * provokes undefined behaviour. 192 * 193 * Return: pointer to the &page (garbage if @netmem is not page-backed). 194 */ 195 static inline struct page *__netmem_to_page(netmem_ref netmem) 196 { 197 return (__force struct page *)netmem; 198 } 199 200 static inline struct page *netmem_to_page(netmem_ref netmem) 201 { 202 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 203 return NULL; 204 205 return __netmem_to_page(netmem); 206 } 207 208 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) 209 { 210 if (netmem_is_net_iov(netmem)) 211 return (struct net_iov *)((__force unsigned long)netmem & 212 ~NET_IOV); 213 214 DEBUG_NET_WARN_ON_ONCE(true); 215 return NULL; 216 } 217 218 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) 219 { 220 return (__force netmem_ref)((unsigned long)niov | NET_IOV); 221 } 222 223 #define page_to_netmem(p) (_Generic((p), \ 224 const struct page * : (__force const netmem_ref)(p), \ 225 struct page * : (__force netmem_ref)(p))) 226 227 /** 228 * virt_to_netmem - convert virtual memory pointer to a netmem reference 229 * @data: host memory pointer to convert 230 * 231 * Return: netmem reference to the &page backing this virtual address. 232 */ 233 static inline netmem_ref virt_to_netmem(const void *data) 234 { 235 return page_to_netmem(virt_to_page(data)); 236 } 237 238 static inline int netmem_ref_count(netmem_ref netmem) 239 { 240 /* The non-pp refcount of net_iov is always 1. On net_iov, we only 241 * support pp refcounting which uses the pp_ref_count field. 242 */ 243 if (netmem_is_net_iov(netmem)) 244 return 1; 245 246 return page_ref_count(netmem_to_page(netmem)); 247 } 248 249 static inline unsigned long netmem_pfn_trace(netmem_ref netmem) 250 { 251 if (netmem_is_net_iov(netmem)) 252 return 0; 253 254 return page_to_pfn(netmem_to_page(netmem)); 255 } 256 257 /* XXX: How to extract netmem_desc from page must be changed, once 258 * netmem_desc no longer overlays on page and will be allocated through 259 * slab. 260 */ 261 #define __pp_page_to_nmdesc(p) (_Generic((p), \ 262 const struct page * : (const struct netmem_desc *)(p), \ 263 struct page * : (struct netmem_desc *)(p))) 264 265 /* CAUTION: Check if the page is a pp page before calling this helper or 266 * know it's a pp page. 267 */ 268 #define pp_page_to_nmdesc(p) \ 269 ({ \ 270 DEBUG_NET_WARN_ON_ONCE(!PageNetpp(p)); \ 271 __pp_page_to_nmdesc(p); \ 272 }) 273 274 /** 275 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing 276 * @netmem 277 * @netmem: netmem reference to convert 278 * 279 * Unsafe version that can be used only when @netmem is always backed by 280 * system memory, performs faster and generates smaller object code (no 281 * check for the LSB, no WARN). When @netmem points to IOV, provokes 282 * undefined behaviour. 283 * 284 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed 285 * by system memory). 286 */ 287 static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem) 288 { 289 return (__force struct netmem_desc *)netmem; 290 } 291 292 /* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for 293 * access to common fields. 294 * @netmem: netmem reference to get netmem_desc. 295 * 296 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same 297 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc. 298 * 299 * Return: the pointer to struct netmem_desc * regardless of its 300 * underlying type. 301 */ 302 static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem) 303 { 304 void *p = (void *)((__force unsigned long)netmem & ~NET_IOV); 305 306 if (netmem_is_net_iov(netmem)) 307 return &((struct net_iov *)p)->desc; 308 309 return __pp_page_to_nmdesc((struct page *)p); 310 } 311 312 /** 313 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 314 * @netmem: netmem reference to get the pointer from 315 * 316 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 317 * e.g. when it's a header buffer, performs faster and generates smaller 318 * object code (avoids clearing the LSB). When @netmem points to IOV, 319 * provokes invalid memory access. 320 * 321 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 322 */ 323 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 324 { 325 return __netmem_to_nmdesc(netmem)->pp; 326 } 327 328 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) 329 { 330 return netmem_to_nmdesc(netmem)->pp; 331 } 332 333 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) 334 { 335 return &netmem_to_nmdesc(netmem)->pp_ref_count; 336 } 337 338 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) 339 { 340 /* NUMA node preference only makes sense if we're allocating 341 * system memory. Memory providers (which give us net_iovs) 342 * choose for us. 343 */ 344 if (netmem_is_net_iov(netmem)) 345 return true; 346 347 return page_to_nid(netmem_to_page(netmem)) == pref_nid; 348 } 349 350 static inline netmem_ref netmem_compound_head(netmem_ref netmem) 351 { 352 /* niov are never compounded */ 353 if (netmem_is_net_iov(netmem)) 354 return netmem; 355 356 return page_to_netmem(compound_head(netmem_to_page(netmem))); 357 } 358 359 /** 360 * __netmem_address - unsafely get pointer to the memory backing @netmem 361 * @netmem: netmem reference to get the pointer for 362 * 363 * Unsafe version of netmem_address(). When @netmem is always page-backed, 364 * e.g. when it's a header buffer, performs faster and generates smaller 365 * object code (no check for the LSB). When @netmem points to IOV, provokes 366 * undefined behaviour. 367 * 368 * Return: pointer to the memory (garbage if @netmem is not page-backed). 369 */ 370 static inline void *__netmem_address(netmem_ref netmem) 371 { 372 return page_address(__netmem_to_page(netmem)); 373 } 374 375 static inline void *netmem_address(netmem_ref netmem) 376 { 377 if (netmem_is_net_iov(netmem)) 378 return NULL; 379 380 return __netmem_address(netmem); 381 } 382 383 /** 384 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 385 * @netmem: netmem reference to check 386 * 387 * Return: true if @netmem is page-backed and the page was allocated under 388 * memory pressure, false otherwise. 389 */ 390 static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 391 { 392 if (netmem_is_net_iov(netmem)) 393 return false; 394 395 return page_is_pfmemalloc(netmem_to_page(netmem)); 396 } 397 398 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) 399 { 400 return netmem_to_nmdesc(netmem)->dma_addr; 401 } 402 403 #if defined(CONFIG_NET_DEVMEM) 404 static inline bool net_is_devmem_iov(const struct net_iov *niov) 405 { 406 return niov->type == NET_IOV_DMABUF; 407 } 408 #else 409 static inline bool net_is_devmem_iov(const struct net_iov *niov) 410 { 411 return false; 412 } 413 #endif 414 415 void __get_netmem(netmem_ref netmem); 416 void __put_netmem(netmem_ref netmem); 417 418 static __always_inline void get_netmem(netmem_ref netmem) 419 { 420 if (netmem_is_net_iov(netmem)) 421 __get_netmem(netmem); 422 else 423 get_page(netmem_to_page(netmem)); 424 } 425 426 static __always_inline void put_netmem(netmem_ref netmem) 427 { 428 if (netmem_is_net_iov(netmem)) 429 __put_netmem(netmem); 430 else 431 put_page(netmem_to_page(netmem)); 432 } 433 434 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ 435 do { \ 436 if (!netmem_is_net_iov(NETMEM)) \ 437 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ 438 else \ 439 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ 440 } while (0) 441 442 static inline void netmem_dma_unmap_page_attrs(struct device *dev, 443 dma_addr_t addr, size_t size, 444 enum dma_data_direction dir, 445 unsigned long attrs) 446 { 447 if (!addr) 448 return; 449 450 dma_unmap_page_attrs(dev, addr, size, dir, attrs); 451 } 452 453 #endif /* _NET_NETMEM_H */ 454