1 /* SPDX-License-Identifier: GPL-2.0
2 *
3 * Network memory
4 *
5 * Author: Mina Almasry <almasrymina@google.com>
6 */
7
8 #ifndef _NET_NETMEM_H
9 #define _NET_NETMEM_H
10
11 #include <linux/dma-mapping.h>
12 #include <linux/mm.h>
13 #include <net/net_debug.h>
14
15 /* These fields in struct page are used by the page_pool and net stack:
16 *
17 * struct {
18 * unsigned long pp_magic;
19 * struct page_pool *pp;
20 * unsigned long _pp_mapping_pad;
21 * unsigned long dma_addr;
22 * atomic_long_t pp_ref_count;
23 * };
24 *
25 * We mirror the page_pool fields here so the page_pool can access these
26 * fields without worrying whether the underlying fields belong to a
27 * page or netmem_desc.
28 *
29 * CAUTION: Do not update the fields in netmem_desc without also
30 * updating the anonymous aliasing union in struct net_iov.
31 */
32 struct netmem_desc {
33 unsigned long _flags;
34 unsigned long pp_magic;
35 struct page_pool *pp;
36 unsigned long _pp_mapping_pad;
37 unsigned long dma_addr;
38 atomic_long_t pp_ref_count;
39 };
40
41 #define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \
42 static_assert(offsetof(struct page, pg) == \
43 offsetof(struct netmem_desc, desc))
44 NETMEM_DESC_ASSERT_OFFSET(flags, _flags);
45 NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic);
46 NETMEM_DESC_ASSERT_OFFSET(pp, pp);
47 NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
48 NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr);
49 NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
50 #undef NETMEM_DESC_ASSERT_OFFSET
51
52 /*
53 * Since struct netmem_desc uses the space in struct page, the size
54 * should be checked, until struct netmem_desc has its own instance from
55 * slab, to avoid conflicting with other members within struct page.
56 */
57 static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount));
58
59 /* net_iov */
60
61 DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers);
62
63 /* We overload the LSB of the struct page pointer to indicate whether it's
64 * a page or net_iov.
65 */
66 #define NET_IOV 0x01UL
67
68 enum net_iov_type {
69 NET_IOV_DMABUF,
70 NET_IOV_IOURING,
71 };
72
73 /* A memory descriptor representing abstract networking I/O vectors,
74 * generally for non-pages memory that doesn't have its corresponding
75 * struct page and needs to be explicitly allocated through slab.
76 *
77 * net_iovs are allocated and used by networking code, and the size of
78 * the chunk is PAGE_SIZE.
79 *
80 * This memory can be any form of non-struct paged memory. Examples
81 * include imported dmabuf memory and imported io_uring memory. See
82 * net_iov_type for all the supported types.
83 *
84 * @pp_magic: pp field, similar to the one in struct page/struct
85 * netmem_desc.
86 * @pp: the pp this net_iov belongs to, if any.
87 * @dma_addr: the dma addrs of the net_iov. Needed for the network
88 * card to send/receive this net_iov.
89 * @pp_ref_count: the pp ref count of this net_iov, exactly the same
90 * usage as struct page/struct netmem_desc.
91 * @owner: the net_iov_area this net_iov belongs to, if any.
92 * @type: the type of the memory. Different types of net_iovs are
93 * supported.
94 */
95 struct net_iov {
96 struct netmem_desc desc;
97 unsigned int page_type;
98 enum net_iov_type type;
99 struct net_iov_area *owner;
100 };
101
102 /* Make sure 'the offset of page_type in struct page == the offset of
103 * type in struct net_iov'.
104 */
105 #define NET_IOV_ASSERT_OFFSET(pg, iov) \
106 static_assert(offsetof(struct page, pg) == \
107 offsetof(struct net_iov, iov))
108 NET_IOV_ASSERT_OFFSET(page_type, page_type);
109 #undef NET_IOV_ASSERT_OFFSET
110
111 struct net_iov_area {
112 /* Array of net_iovs for this area. */
113 struct net_iov *niovs;
114 size_t num_niovs;
115
116 /* Offset into the dma-buf where this chunk starts. */
117 unsigned long base_virtual;
118 };
119
net_iov_owner(const struct net_iov * niov)120 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov)
121 {
122 return niov->owner;
123 }
124
net_iov_idx(const struct net_iov * niov)125 static inline unsigned int net_iov_idx(const struct net_iov *niov)
126 {
127 return niov - net_iov_owner(niov)->niovs;
128 }
129
130 /* Initialize a niov: stamp the owning area, the memory provider type,
131 * and the page_type "no type" sentinel expected by the page-type API
132 * (see PAGE_TYPE_OPS in <linux/page-flags.h>) so that
133 * page_pool_set_pp_info() can later call __SetPageNetpp() on a niov
134 * cast to struct page.
135 */
net_iov_init(struct net_iov * niov,struct net_iov_area * owner,enum net_iov_type type)136 static inline void net_iov_init(struct net_iov *niov,
137 struct net_iov_area *owner,
138 enum net_iov_type type)
139 {
140 niov->owner = owner;
141 niov->type = type;
142 niov->page_type = UINT_MAX;
143 }
144
145 /* netmem */
146
147 /**
148 * typedef netmem_ref - a nonexistent type marking a reference to generic
149 * network memory.
150 *
151 * A netmem_ref can be a struct page* or a struct net_iov* underneath.
152 *
153 * Use the supplied helpers to obtain the underlying memory pointer and fields.
154 */
155 typedef unsigned long __bitwise netmem_ref;
156
netmem_is_net_iov(const netmem_ref netmem)157 static inline bool netmem_is_net_iov(const netmem_ref netmem)
158 {
159 return (__force unsigned long)netmem & NET_IOV;
160 }
161
162 /**
163 * __netmem_to_page - unsafely get pointer to the &page backing @netmem
164 * @netmem: netmem reference to convert
165 *
166 * Unsafe version of netmem_to_page(). When @netmem is always page-backed,
167 * e.g. when it's a header buffer, performs faster and generates smaller
168 * object code (no check for the LSB, no WARN). When @netmem points to IOV,
169 * provokes undefined behaviour.
170 *
171 * Return: pointer to the &page (garbage if @netmem is not page-backed).
172 */
__netmem_to_page(netmem_ref netmem)173 static inline struct page *__netmem_to_page(netmem_ref netmem)
174 {
175 return (__force struct page *)netmem;
176 }
177
netmem_to_page(netmem_ref netmem)178 static inline struct page *netmem_to_page(netmem_ref netmem)
179 {
180 if (WARN_ON_ONCE(netmem_is_net_iov(netmem)))
181 return NULL;
182
183 return __netmem_to_page(netmem);
184 }
185
netmem_to_net_iov(netmem_ref netmem)186 static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem)
187 {
188 if (netmem_is_net_iov(netmem))
189 return (struct net_iov *)((__force unsigned long)netmem &
190 ~NET_IOV);
191
192 DEBUG_NET_WARN_ON_ONCE(true);
193 return NULL;
194 }
195
net_iov_to_netmem(struct net_iov * niov)196 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov)
197 {
198 return (__force netmem_ref)((unsigned long)niov | NET_IOV);
199 }
200
201 #define page_to_netmem(p) (_Generic((p), \
202 const struct page * : (__force const netmem_ref)(p), \
203 struct page * : (__force netmem_ref)(p)))
204
205 /**
206 * virt_to_netmem - convert virtual memory pointer to a netmem reference
207 * @data: host memory pointer to convert
208 *
209 * Return: netmem reference to the &page backing this virtual address.
210 */
virt_to_netmem(const void * data)211 static inline netmem_ref virt_to_netmem(const void *data)
212 {
213 return page_to_netmem(virt_to_page(data));
214 }
215
netmem_ref_count(netmem_ref netmem)216 static inline int netmem_ref_count(netmem_ref netmem)
217 {
218 /* The non-pp refcount of net_iov is always 1. On net_iov, we only
219 * support pp refcounting which uses the pp_ref_count field.
220 */
221 if (netmem_is_net_iov(netmem))
222 return 1;
223
224 return page_ref_count(netmem_to_page(netmem));
225 }
226
netmem_pfn_trace(netmem_ref netmem)227 static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
228 {
229 if (netmem_is_net_iov(netmem))
230 return 0;
231
232 return page_to_pfn(netmem_to_page(netmem));
233 }
234
235 /* XXX: How to extract netmem_desc from page must be changed, once
236 * netmem_desc no longer overlays on page and will be allocated through
237 * slab.
238 */
239 #define __pp_page_to_nmdesc(p) (_Generic((p), \
240 const struct page * : (const struct netmem_desc *)(p), \
241 struct page * : (struct netmem_desc *)(p)))
242
243 /* CAUTION: Check if the page is a pp page before calling this helper or
244 * know it's a pp page.
245 */
246 #define pp_page_to_nmdesc(p) \
247 ({ \
248 DEBUG_NET_WARN_ON_ONCE(!PageNetpp(p)); \
249 __pp_page_to_nmdesc(p); \
250 })
251
252 /**
253 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing
254 * @netmem
255 * @netmem: netmem reference to convert
256 *
257 * Unsafe version that can be used only when @netmem is always backed by
258 * system memory, performs faster and generates smaller object code (no
259 * check for the LSB, no WARN). When @netmem points to IOV, provokes
260 * undefined behaviour.
261 *
262 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed
263 * by system memory).
264 */
__netmem_to_nmdesc(netmem_ref netmem)265 static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem)
266 {
267 return (__force struct netmem_desc *)netmem;
268 }
269
270 /* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for
271 * access to common fields.
272 * @netmem: netmem reference to get netmem_desc.
273 *
274 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same
275 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc.
276 *
277 * Return: the pointer to struct netmem_desc * regardless of its
278 * underlying type.
279 */
netmem_to_nmdesc(netmem_ref netmem)280 static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem)
281 {
282 void *p = (void *)((__force unsigned long)netmem & ~NET_IOV);
283
284 if (netmem_is_net_iov(netmem))
285 return &((struct net_iov *)p)->desc;
286
287 return __pp_page_to_nmdesc((struct page *)p);
288 }
289
290 /**
291 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem
292 * @netmem: netmem reference to get the pointer from
293 *
294 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed,
295 * e.g. when it's a header buffer, performs faster and generates smaller
296 * object code (avoids clearing the LSB). When @netmem points to IOV,
297 * provokes invalid memory access.
298 *
299 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed).
300 */
__netmem_get_pp(netmem_ref netmem)301 static inline struct page_pool *__netmem_get_pp(netmem_ref netmem)
302 {
303 return __netmem_to_nmdesc(netmem)->pp;
304 }
305
netmem_get_pp(netmem_ref netmem)306 static inline struct page_pool *netmem_get_pp(netmem_ref netmem)
307 {
308 return netmem_to_nmdesc(netmem)->pp;
309 }
310
netmem_get_pp_ref_count_ref(netmem_ref netmem)311 static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem)
312 {
313 return &netmem_to_nmdesc(netmem)->pp_ref_count;
314 }
315
netmem_is_pref_nid(netmem_ref netmem,int pref_nid)316 static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid)
317 {
318 /* NUMA node preference only makes sense if we're allocating
319 * system memory. Memory providers (which give us net_iovs)
320 * choose for us.
321 */
322 if (netmem_is_net_iov(netmem))
323 return true;
324
325 return page_to_nid(netmem_to_page(netmem)) == pref_nid;
326 }
327
netmem_compound_head(netmem_ref netmem)328 static inline netmem_ref netmem_compound_head(netmem_ref netmem)
329 {
330 /* niov are never compounded */
331 if (netmem_is_net_iov(netmem))
332 return netmem;
333
334 return page_to_netmem(compound_head(netmem_to_page(netmem)));
335 }
336
337 /**
338 * __netmem_address - unsafely get pointer to the memory backing @netmem
339 * @netmem: netmem reference to get the pointer for
340 *
341 * Unsafe version of netmem_address(). When @netmem is always page-backed,
342 * e.g. when it's a header buffer, performs faster and generates smaller
343 * object code (no check for the LSB). When @netmem points to IOV, provokes
344 * undefined behaviour.
345 *
346 * Return: pointer to the memory (garbage if @netmem is not page-backed).
347 */
__netmem_address(netmem_ref netmem)348 static inline void *__netmem_address(netmem_ref netmem)
349 {
350 return page_address(__netmem_to_page(netmem));
351 }
352
netmem_address(netmem_ref netmem)353 static inline void *netmem_address(netmem_ref netmem)
354 {
355 if (netmem_is_net_iov(netmem))
356 return NULL;
357
358 return __netmem_address(netmem);
359 }
360
361 /**
362 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure
363 * @netmem: netmem reference to check
364 *
365 * Return: true if @netmem is page-backed and the page was allocated under
366 * memory pressure, false otherwise.
367 */
netmem_is_pfmemalloc(netmem_ref netmem)368 static inline bool netmem_is_pfmemalloc(netmem_ref netmem)
369 {
370 if (netmem_is_net_iov(netmem))
371 return false;
372
373 return page_is_pfmemalloc(netmem_to_page(netmem));
374 }
375
netmem_get_dma_addr(netmem_ref netmem)376 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
377 {
378 return netmem_to_nmdesc(netmem)->dma_addr;
379 }
380
381 #if defined(CONFIG_NET_DEVMEM)
net_is_devmem_iov(const struct net_iov * niov)382 static inline bool net_is_devmem_iov(const struct net_iov *niov)
383 {
384 return niov->type == NET_IOV_DMABUF;
385 }
386 #else
net_is_devmem_iov(const struct net_iov * niov)387 static inline bool net_is_devmem_iov(const struct net_iov *niov)
388 {
389 return false;
390 }
391 #endif
392
393 void __get_netmem(netmem_ref netmem);
394 void __put_netmem(netmem_ref netmem);
395
get_netmem(netmem_ref netmem)396 static __always_inline void get_netmem(netmem_ref netmem)
397 {
398 if (netmem_is_net_iov(netmem))
399 __get_netmem(netmem);
400 else
401 get_page(netmem_to_page(netmem));
402 }
403
put_netmem(netmem_ref netmem)404 static __always_inline void put_netmem(netmem_ref netmem)
405 {
406 if (netmem_is_net_iov(netmem))
407 __put_netmem(netmem);
408 else
409 put_page(netmem_to_page(netmem));
410 }
411
412 #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \
413 do { \
414 if (!netmem_is_net_iov(NETMEM)) \
415 dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \
416 else \
417 dma_unmap_addr_set(PTR, ADDR_NAME, 0); \
418 } while (0)
419
netmem_dma_unmap_page_attrs(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)420 static inline void netmem_dma_unmap_page_attrs(struct device *dev,
421 dma_addr_t addr, size_t size,
422 enum dma_data_direction dir,
423 unsigned long attrs)
424 {
425 if (!addr)
426 return;
427
428 dma_unmap_page_attrs(dev, addr, size, dir, attrs);
429 }
430
431 #endif /* _NET_NETMEM_H */
432