1 /* SPDX-License-Identifier: GPL-2.0 2 * 3 * Network memory 4 * 5 * Author: Mina Almasry <almasrymina@google.com> 6 */ 7 8 #ifndef _NET_NETMEM_H 9 #define _NET_NETMEM_H 10 11 #include <linux/dma-mapping.h> 12 #include <linux/mm.h> 13 #include <net/net_debug.h> 14 15 /* These fields in struct page are used by the page_pool and net stack: 16 * 17 * struct { 18 * unsigned long pp_magic; 19 * struct page_pool *pp; 20 * unsigned long _pp_mapping_pad; 21 * unsigned long dma_addr; 22 * atomic_long_t pp_ref_count; 23 * }; 24 * 25 * We mirror the page_pool fields here so the page_pool can access these 26 * fields without worrying whether the underlying fields belong to a 27 * page or netmem_desc. 28 * 29 * CAUTION: Do not update the fields in netmem_desc without also 30 * updating the anonymous aliasing union in struct net_iov. 31 */ 32 struct netmem_desc { 33 unsigned long _flags; 34 unsigned long pp_magic; 35 struct page_pool *pp; 36 unsigned long _pp_mapping_pad; 37 unsigned long dma_addr; 38 atomic_long_t pp_ref_count; 39 }; 40 41 #define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \ 42 static_assert(offsetof(struct page, pg) == \ 43 offsetof(struct netmem_desc, desc)) 44 NETMEM_DESC_ASSERT_OFFSET(flags, _flags); 45 NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic); 46 NETMEM_DESC_ASSERT_OFFSET(pp, pp); 47 NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 48 NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr); 49 NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 50 #undef NETMEM_DESC_ASSERT_OFFSET 51 52 /* 53 * Since struct netmem_desc uses the space in struct page, the size 54 * should be checked, until struct netmem_desc has its own instance from 55 * slab, to avoid conflicting with other members within struct page. 56 */ 57 static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount)); 58 59 /* net_iov */ 60 61 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers); 62 63 /* We overload the LSB of the struct page pointer to indicate whether it's 64 * a page or net_iov. 65 */ 66 #define NET_IOV 0x01UL 67 68 enum net_iov_type { 69 NET_IOV_DMABUF, 70 NET_IOV_IOURING, 71 72 /* Force size to unsigned long to make the NET_IOV_ASSERTS below pass. 73 */ 74 NET_IOV_MAX = ULONG_MAX 75 }; 76 77 /* A memory descriptor representing abstract networking I/O vectors, 78 * generally for non-pages memory that doesn't have its corresponding 79 * struct page and needs to be explicitly allocated through slab. 80 * 81 * net_iovs are allocated and used by networking code, and the size of 82 * the chunk is PAGE_SIZE. 83 * 84 * This memory can be any form of non-struct paged memory. Examples 85 * include imported dmabuf memory and imported io_uring memory. See 86 * net_iov_type for all the supported types. 87 * 88 * @pp_magic: pp field, similar to the one in struct page/struct 89 * netmem_desc. 90 * @pp: the pp this net_iov belongs to, if any. 91 * @dma_addr: the dma addrs of the net_iov. Needed for the network 92 * card to send/receive this net_iov. 93 * @pp_ref_count: the pp ref count of this net_iov, exactly the same 94 * usage as struct page/struct netmem_desc. 95 * @owner: the net_iov_area this net_iov belongs to, if any. 96 * @type: the type of the memory. Different types of net_iovs are 97 * supported. 98 */ 99 struct net_iov { 100 union { 101 struct netmem_desc desc; 102 103 /* XXX: The following part should be removed once all 104 * the references to them are converted so as to be 105 * accessed via netmem_desc e.g. niov->desc.pp instead 106 * of niov->pp. 107 */ 108 struct { 109 unsigned long _flags; 110 unsigned long pp_magic; 111 struct page_pool *pp; 112 unsigned long _pp_mapping_pad; 113 unsigned long dma_addr; 114 atomic_long_t pp_ref_count; 115 }; 116 }; 117 struct net_iov_area *owner; 118 enum net_iov_type type; 119 }; 120 121 struct net_iov_area { 122 /* Array of net_iovs for this area. */ 123 struct net_iov *niovs; 124 size_t num_niovs; 125 126 /* Offset into the dma-buf where this chunk starts. */ 127 unsigned long base_virtual; 128 }; 129 130 /* net_iov is union'ed with struct netmem_desc mirroring struct page, so 131 * the page_pool can access these fields without worrying whether the 132 * underlying fields are accessed via netmem_desc or directly via 133 * net_iov, until all the references to them are converted so as to be 134 * accessed via netmem_desc e.g. niov->desc.pp instead of niov->pp. 135 * 136 * The non-net stack fields of struct page are private to the mm stack 137 * and must never be mirrored to net_iov. 138 */ 139 #define NET_IOV_ASSERT_OFFSET(desc, iov) \ 140 static_assert(offsetof(struct netmem_desc, desc) == \ 141 offsetof(struct net_iov, iov)) 142 NET_IOV_ASSERT_OFFSET(_flags, _flags); 143 NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic); 144 NET_IOV_ASSERT_OFFSET(pp, pp); 145 NET_IOV_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); 146 NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr); 147 NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count); 148 #undef NET_IOV_ASSERT_OFFSET 149 150 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) 151 { 152 return niov->owner; 153 } 154 155 static inline unsigned int net_iov_idx(const struct net_iov *niov) 156 { 157 return niov - net_iov_owner(niov)->niovs; 158 } 159 160 /* netmem */ 161 162 /** 163 * typedef netmem_ref - a nonexistent type marking a reference to generic 164 * network memory. 165 * 166 * A netmem_ref can be a struct page* or a struct net_iov* underneath. 167 * 168 * Use the supplied helpers to obtain the underlying memory pointer and fields. 169 */ 170 typedef unsigned long __bitwise netmem_ref; 171 172 static inline bool netmem_is_net_iov(const netmem_ref netmem) 173 { 174 return (__force unsigned long)netmem & NET_IOV; 175 } 176 177 /** 178 * __netmem_to_page - unsafely get pointer to the &page backing @netmem 179 * @netmem: netmem reference to convert 180 * 181 * Unsafe version of netmem_to_page(). When @netmem is always page-backed, 182 * e.g. when it's a header buffer, performs faster and generates smaller 183 * object code (no check for the LSB, no WARN). When @netmem points to IOV, 184 * provokes undefined behaviour. 185 * 186 * Return: pointer to the &page (garbage if @netmem is not page-backed). 187 */ 188 static inline struct page *__netmem_to_page(netmem_ref netmem) 189 { 190 return (__force struct page *)netmem; 191 } 192 193 static inline struct page *netmem_to_page(netmem_ref netmem) 194 { 195 if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) 196 return NULL; 197 198 return __netmem_to_page(netmem); 199 } 200 201 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem) 202 { 203 if (netmem_is_net_iov(netmem)) 204 return (struct net_iov *)((__force unsigned long)netmem & 205 ~NET_IOV); 206 207 DEBUG_NET_WARN_ON_ONCE(true); 208 return NULL; 209 } 210 211 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) 212 { 213 return (__force netmem_ref)((unsigned long)niov | NET_IOV); 214 } 215 216 #define page_to_netmem(p) (_Generic((p), \ 217 const struct page * : (__force const netmem_ref)(p), \ 218 struct page * : (__force netmem_ref)(p))) 219 220 /** 221 * virt_to_netmem - convert virtual memory pointer to a netmem reference 222 * @data: host memory pointer to convert 223 * 224 * Return: netmem reference to the &page backing this virtual address. 225 */ 226 static inline netmem_ref virt_to_netmem(const void *data) 227 { 228 return page_to_netmem(virt_to_page(data)); 229 } 230 231 static inline int netmem_ref_count(netmem_ref netmem) 232 { 233 /* The non-pp refcount of net_iov is always 1. On net_iov, we only 234 * support pp refcounting which uses the pp_ref_count field. 235 */ 236 if (netmem_is_net_iov(netmem)) 237 return 1; 238 239 return page_ref_count(netmem_to_page(netmem)); 240 } 241 242 static inline unsigned long netmem_pfn_trace(netmem_ref netmem) 243 { 244 if (netmem_is_net_iov(netmem)) 245 return 0; 246 247 return page_to_pfn(netmem_to_page(netmem)); 248 } 249 250 /** 251 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing 252 * @netmem 253 * @netmem: netmem reference to convert 254 * 255 * Unsafe version that can be used only when @netmem is always backed by 256 * system memory, performs faster and generates smaller object code (no 257 * check for the LSB, no WARN). When @netmem points to IOV, provokes 258 * undefined behaviour. 259 * 260 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed 261 * by system memory). 262 */ 263 static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem) 264 { 265 return (__force struct netmem_desc *)netmem; 266 } 267 268 /* __netmem_clear_lsb - convert netmem_ref to struct net_iov * for access to 269 * common fields. 270 * @netmem: netmem reference to extract as net_iov. 271 * 272 * All the sub types of netmem_ref (page, net_iov) have the same pp, pp_magic, 273 * dma_addr, and pp_ref_count fields at the same offsets. Thus, we can access 274 * these fields without a type check to make sure that the underlying mem is 275 * net_iov or page. 276 * 277 * The resulting value of this function can only be used to access the fields 278 * that are NET_IOV_ASSERT_OFFSET'd. Accessing any other fields will result in 279 * undefined behavior. 280 * 281 * Return: the netmem_ref cast to net_iov* regardless of its underlying type. 282 */ 283 static inline struct net_iov *__netmem_clear_lsb(netmem_ref netmem) 284 { 285 return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV); 286 } 287 288 /* XXX: How to extract netmem_desc from page must be changed, once 289 * netmem_desc no longer overlays on page and will be allocated through 290 * slab. 291 */ 292 #define __pp_page_to_nmdesc(p) (_Generic((p), \ 293 const struct page * : (const struct netmem_desc *)(p), \ 294 struct page * : (struct netmem_desc *)(p))) 295 296 /* CAUTION: Check if the page is a pp page before calling this helper or 297 * know it's a pp page. 298 */ 299 #define pp_page_to_nmdesc(p) \ 300 ({ \ 301 DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p)); \ 302 __pp_page_to_nmdesc(p); \ 303 }) 304 305 /** 306 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem 307 * @netmem: netmem reference to get the pointer from 308 * 309 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, 310 * e.g. when it's a header buffer, performs faster and generates smaller 311 * object code (avoids clearing the LSB). When @netmem points to IOV, 312 * provokes invalid memory access. 313 * 314 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). 315 */ 316 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem) 317 { 318 return __netmem_to_nmdesc(netmem)->pp; 319 } 320 321 static inline struct page_pool *netmem_get_pp(netmem_ref netmem) 322 { 323 return __netmem_clear_lsb(netmem)->pp; 324 } 325 326 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem) 327 { 328 return &__netmem_clear_lsb(netmem)->pp_ref_count; 329 } 330 331 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid) 332 { 333 /* NUMA node preference only makes sense if we're allocating 334 * system memory. Memory providers (which give us net_iovs) 335 * choose for us. 336 */ 337 if (netmem_is_net_iov(netmem)) 338 return true; 339 340 return page_to_nid(netmem_to_page(netmem)) == pref_nid; 341 } 342 343 static inline netmem_ref netmem_compound_head(netmem_ref netmem) 344 { 345 /* niov are never compounded */ 346 if (netmem_is_net_iov(netmem)) 347 return netmem; 348 349 return page_to_netmem(compound_head(netmem_to_page(netmem))); 350 } 351 352 /** 353 * __netmem_address - unsafely get pointer to the memory backing @netmem 354 * @netmem: netmem reference to get the pointer for 355 * 356 * Unsafe version of netmem_address(). When @netmem is always page-backed, 357 * e.g. when it's a header buffer, performs faster and generates smaller 358 * object code (no check for the LSB). When @netmem points to IOV, provokes 359 * undefined behaviour. 360 * 361 * Return: pointer to the memory (garbage if @netmem is not page-backed). 362 */ 363 static inline void *__netmem_address(netmem_ref netmem) 364 { 365 return page_address(__netmem_to_page(netmem)); 366 } 367 368 static inline void *netmem_address(netmem_ref netmem) 369 { 370 if (netmem_is_net_iov(netmem)) 371 return NULL; 372 373 return __netmem_address(netmem); 374 } 375 376 /** 377 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure 378 * @netmem: netmem reference to check 379 * 380 * Return: true if @netmem is page-backed and the page was allocated under 381 * memory pressure, false otherwise. 382 */ 383 static inline bool netmem_is_pfmemalloc(netmem_ref netmem) 384 { 385 if (netmem_is_net_iov(netmem)) 386 return false; 387 388 return page_is_pfmemalloc(netmem_to_page(netmem)); 389 } 390 391 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) 392 { 393 return __netmem_clear_lsb(netmem)->dma_addr; 394 } 395 396 void get_netmem(netmem_ref netmem); 397 void put_netmem(netmem_ref netmem); 398 399 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ 400 do { \ 401 if (!netmem_is_net_iov(NETMEM)) \ 402 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ 403 else \ 404 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ 405 } while (0) 406 407 static inline void netmem_dma_unmap_page_attrs(struct device *dev, 408 dma_addr_t addr, size_t size, 409 enum dma_data_direction dir, 410 unsigned long attrs) 411 { 412 if (!addr) 413 return; 414 415 dma_unmap_page_attrs(dev, addr, size, dir, attrs); 416 } 417 418 #endif /* _NET_NETMEM_H */ 419