1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Network memory 4 * 5 * Author: Mina Almasry <almasrymina@google.com> 6 */ 7 8 #ifndef _NET_NETMEM_H 9 #define _NET_NETMEM_H 10 11 #include <linux/dma-mapping.h> 12 #include <linux/mm.h> 13 #include <net/net_debug.h> 14 15 /* These fields in struct page are used by the page_pool and net stack: 16 * 17 * struct { 18 * unsigned long pp_magic; 19 * struct page_pool *pp; 20 * unsigned long _pp_mapping_pad; 21 * unsigned long dma_addr; 22 * atomic_long_t pp_ref_count; 23 * }; 24 * 25 * We mirror the page_pool fields here so the page_pool can access these 26 * fields without worrying whether the underlying fields belong to a 27 * page or netmem_desc. 28 * 29 * CAUTION: Do not update the fields in netmem_desc without also 30 * updating the anonymous aliasing union in struct net_iov. 31 */ 32 struct netmem_desc { 33 unsigned long _flags; 34 unsigned long pp_magic; 35 struct page_pool *pp; 36 unsigned long _pp_mapping_pad; 37 unsigned long dma_addr; 38 atomic_long_t pp_ref_count; 39 }; 40 41 #define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \ 42 static_assert(offsetof(struct page, pg) == \ 43 offsetof(struct netmem_desc, desc)) 44 NETMEM_DESC_ASSERT_OFFSET(flags, _flags); 45 NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic); 46 NETMEM_DESC_ASSERT_OFFSET(pp, pp); 47 NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 48 NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr); 49 NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 50 #undef NETMEM_DESC_ASSERT_OFFSET 51 52 /* 53 * Since struct netmem_desc uses the space in struct page, the size 54 * should be checked, until struct netmem_desc has its own instance from 55 * slab, to avoid conflicting with other members within struct page. 56 */ 57 static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount)); 58 59 /* net_iov */ 60 61 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); 62 63 /* We overload the LSB of the struct page pointer to indicate whether it's 64 * a page or net_iov. 65 */ 66 #define NET_IOV 0x01UL 67 68 enum net_iov_type { 69 NET_IOV_DMABUF, 70 NET_IOV_IOURING, 71 }; 72 73 /* A memory descriptor representing abstract networking I/O vectors, 74 * generally for non-pages memory that doesn't have its corresponding 75 * struct page and needs to be explicitly allocated through slab. 76 * 77 * net_iovs are allocated and used by networking code, and the size of 78 * the chunk is PAGE_SIZE. 79 * 80 * This memory can be any form of non-struct paged memory. Examples 81 * include imported dmabuf memory and imported io_uring memory. See 82 * net_iov_type for all the supported types. 83 * 84 * @pp_magic: pp field, similar to the one in struct page/struct 85 * netmem_desc. 86 * @pp: the pp this net_iov belongs to, if any. 87 * @dma_addr: the dma addrs of the net_iov. Needed for the network 88 * card to send/receive this net_iov. 89 * @pp_ref_count: the pp ref count of this net_iov, exactly the same 90 * usage as struct page/struct netmem_desc. 91 * @owner: the net_iov_area this net_iov belongs to, if any. 92 * @type: the type of the memory. Different types of net_iovs are 93 * supported. 94 */ 95 struct net_iov { 96 struct netmem_desc desc; 97 unsigned int page_type; 98 enum net_iov_type type; 99 struct net_iov_area *owner; 100 }; 101 102 /* Make sure 'the offset of page_type in struct page == the offset of 103 * type in struct net_iov'. 104 */ 105 #define NET_IOV_ASSERT_OFFSET(pg, iov) \ 106 static_assert(offsetof(struct page, pg) == \ 107 offsetof(struct net_iov, iov)) 108 NET_IOV_ASSERT_OFFSET(page_type, page_type); 109 #undef NET_IOV_ASSERT_OFFSET 110 111 struct net_iov_area { 112 /* Array of net_iovs for this area. */ 113 struct net_iov *niovs; 114 size_t num_niovs; 115 116 /* Offset into the dma-buf where this chunk starts. */ 117 unsigned long base_virtual; 118 }; 119 120 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) 121 { 122 return niov->owner; 123 } 124 125 static inline unsigned int net_iov_idx(const struct net_iov *niov) 126 { 127 return niov - net_iov_owner(niov)->niovs; 128 } 129 130 /* Initialize a niov: stamp the owning area, the memory provider type, 131 * and the page_type "no type" sentinel expected by the page-type API 132 * (see PAGE_TYPE_OPS in <linux/page-flags.h>) so that 133 * page_pool_set_pp_info() can later call __SetPageNetpp() on a niov 134 * cast to struct page. 135 */ 136 static inline void net_iov_init(struct net_iov *niov, 137 struct net_iov_area *owner, 138 enum net_iov_type type) 139 { 140 niov->owner = owner; 141 niov->type = type; 142 niov->page_type = UINT_MAX; 143 } 144 145 /* netmem */ 146 147 /** 148 * typedef netmem_ref - a nonexistent type marking a reference to generic 149 * network memory. 150 * 151 * A netmem_ref can be a struct page* or a struct net_iov* underneath. 152 * 153 * Use the supplied helpers to obtain the underlying memory pointer and fields. 154 */ 155 typedef unsigned long __bitwise netmem_ref; 156 157 static inline bool netmem_is_net_iov(const netmem_ref netmem) 158 { 159 return (__force unsigned long)netmem & NET_IOV; 160 } 161 162 /** 163 * __netmem_to_page - unsafely get pointer to the &page backing @netmem 164 * @netmem: netmem reference to convert 165 * 166 * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 167 * e.g. when it's a header buffer, performs faster and generates smaller 168 * object code (no check for the LSB, no WARN). When @netmem points to IOV, 169 * provokes undefined behaviour. 170 * 171 * Return: pointer to the &page (garbage if @netmem is not page-backed). 172 */ 173 static inline struct page *__netmem_to_page(netmem_ref netmem) 174 { 175 return (__force struct page *)netmem; 176 } 177 178 static inline struct page *netmem_to_page(netmem_ref netmem) 179 { 180 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 181 return NULL; 182 183 return __netmem_to_page(netmem); 184 } 185 186 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) 187 { 188 if (netmem_is_net_iov(netmem)) 189 return (struct net_iov *)((__force unsigned long)netmem & 190 ~NET_IOV); 191 192 DEBUG_NET_WARN_ON_ONCE(true); 193 return NULL; 194 } 195 196 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) 197 { 198 return (__force netmem_ref)((unsigned long)niov | NET_IOV); 199 } 200 201 #define page_to_netmem(p) (_Generic((p), \ 202 const struct page * : (__force const netmem_ref)(p), \ 203 struct page * : (__force netmem_ref)(p))) 204 205 /** 206 * virt_to_netmem - convert virtual memory pointer to a netmem reference 207 * @data: host memory pointer to convert 208 * 209 * Return: netmem reference to the &page backing this virtual address. 210 */ 211 static inline netmem_ref virt_to_netmem(const void *data) 212 { 213 return page_to_netmem(virt_to_page(data)); 214 } 215 216 static inline int netmem_ref_count(netmem_ref netmem) 217 { 218 /* The non-pp refcount of net_iov is always 1. On net_iov, we only 219 * support pp refcounting which uses the pp_ref_count field. 220 */ 221 if (netmem_is_net_iov(netmem)) 222 return 1; 223 224 return page_ref_count(netmem_to_page(netmem)); 225 } 226 227 static inline unsigned long netmem_pfn_trace(netmem_ref netmem) 228 { 229 if (netmem_is_net_iov(netmem)) 230 return 0; 231 232 return page_to_pfn(netmem_to_page(netmem)); 233 } 234 235 /* XXX: How to extract netmem_desc from page must be changed, once 236 * netmem_desc no longer overlays on page and will be allocated through 237 * slab. 238 */ 239 #define __pp_page_to_nmdesc(p) (_Generic((p), \ 240 const struct page * : (const struct netmem_desc *)(p), \ 241 struct page * : (struct netmem_desc *)(p))) 242 243 /* CAUTION: Check if the page is a pp page before calling this helper or 244 * know it's a pp page. 245 */ 246 #define pp_page_to_nmdesc(p) \ 247 ({ \ 248 DEBUG_NET_WARN_ON_ONCE(!PageNetpp(p)); \ 249 __pp_page_to_nmdesc(p); \ 250 }) 251 252 /** 253 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing 254 * @netmem 255 * @netmem: netmem reference to convert 256 * 257 * Unsafe version that can be used only when @netmem is always backed by 258 * system memory, performs faster and generates smaller object code (no 259 * check for the LSB, no WARN). When @netmem points to IOV, provokes 260 * undefined behaviour. 261 * 262 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed 263 * by system memory). 264 */ 265 static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem) 266 { 267 return (__force struct netmem_desc *)netmem; 268 } 269 270 /* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for 271 * access to common fields. 272 * @netmem: netmem reference to get netmem_desc. 273 * 274 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same 275 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc. 276 * 277 * Return: the pointer to struct netmem_desc * regardless of its 278 * underlying type. 279 */ 280 static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem) 281 { 282 void *p = (void *)((__force unsigned long)netmem & ~NET_IOV); 283 284 if (netmem_is_net_iov(netmem)) 285 return &((struct net_iov *)p)->desc; 286 287 return __pp_page_to_nmdesc((struct page *)p); 288 } 289 290 /** 291 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 292 * @netmem: netmem reference to get the pointer from 293 * 294 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 295 * e.g. when it's a header buffer, performs faster and generates smaller 296 * object code (avoids clearing the LSB). When @netmem points to IOV, 297 * provokes invalid memory access. 298 * 299 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 300 */ 301 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 302 { 303 return __netmem_to_nmdesc(netmem)->pp; 304 } 305 306 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) 307 { 308 return netmem_to_nmdesc(netmem)->pp; 309 } 310 311 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) 312 { 313 return &netmem_to_nmdesc(netmem)->pp_ref_count; 314 } 315 316 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) 317 { 318 /* NUMA node preference only makes sense if we're allocating 319 * system memory. Memory providers (which give us net_iovs) 320 * choose for us. 321 */ 322 if (netmem_is_net_iov(netmem)) 323 return true; 324 325 return page_to_nid(netmem_to_page(netmem)) == pref_nid; 326 } 327 328 static inline netmem_ref netmem_compound_head(netmem_ref netmem) 329 { 330 /* niov are never compounded */ 331 if (netmem_is_net_iov(netmem)) 332 return netmem; 333 334 return page_to_netmem(compound_head(netmem_to_page(netmem))); 335 } 336 337 /** 338 * __netmem_address - unsafely get pointer to the memory backing @netmem 339 * @netmem: netmem reference to get the pointer for 340 * 341 * Unsafe version of netmem_address(). When @netmem is always page-backed, 342 * e.g. when it's a header buffer, performs faster and generates smaller 343 * object code (no check for the LSB). When @netmem points to IOV, provokes 344 * undefined behaviour. 345 * 346 * Return: pointer to the memory (garbage if @netmem is not page-backed). 347 */ 348 static inline void *__netmem_address(netmem_ref netmem) 349 { 350 return page_address(__netmem_to_page(netmem)); 351 } 352 353 static inline void *netmem_address(netmem_ref netmem) 354 { 355 if (netmem_is_net_iov(netmem)) 356 return NULL; 357 358 return __netmem_address(netmem); 359 } 360 361 /** 362 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 363 * @netmem: netmem reference to check 364 * 365 * Return: true if @netmem is page-backed and the page was allocated under 366 * memory pressure, false otherwise. 367 */ 368 static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 369 { 370 if (netmem_is_net_iov(netmem)) 371 return false; 372 373 return page_is_pfmemalloc(netmem_to_page(netmem)); 374 } 375 376 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) 377 { 378 return netmem_to_nmdesc(netmem)->dma_addr; 379 } 380 381 #if defined(CONFIG_NET_DEVMEM) 382 static inline bool net_is_devmem_iov(const struct net_iov *niov) 383 { 384 return niov->type == NET_IOV_DMABUF; 385 } 386 #else 387 static inline bool net_is_devmem_iov(const struct net_iov *niov) 388 { 389 return false; 390 } 391 #endif 392 393 void __get_netmem(netmem_ref netmem); 394 void __put_netmem(netmem_ref netmem); 395 396 static __always_inline void get_netmem(netmem_ref netmem) 397 { 398 if (netmem_is_net_iov(netmem)) 399 __get_netmem(netmem); 400 else 401 get_page(netmem_to_page(netmem)); 402 } 403 404 static __always_inline void put_netmem(netmem_ref netmem) 405 { 406 if (netmem_is_net_iov(netmem)) 407 __put_netmem(netmem); 408 else 409 put_page(netmem_to_page(netmem)); 410 } 411 412 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ 413 do { \ 414 if (!netmem_is_net_iov(NETMEM)) \ 415 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ 416 else \ 417 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ 418 } while (0) 419 420 static inline void netmem_dma_unmap_page_attrs(struct device *dev, 421 dma_addr_t addr, size_t size, 422 enum dma_data_direction dir, 423 unsigned long attrs) 424 { 425 if (!addr) 426 return; 427 428 dma_unmap_page_attrs(dev, addr, size, dir, attrs); 429 } 430 431 #endif /* _NET_NETMEM_H */ 432