xref: /linux/net/core/page_pool_user.c (revision 0f921404689398943257793f7240db239a23b609)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <linux/mutex.h>
4 #include <linux/netdevice.h>
5 #include <linux/xarray.h>
6 #include <net/net_debug.h>
7 #include <net/netdev_rx_queue.h>
8 #include <net/page_pool/helpers.h>
9 #include <net/page_pool/types.h>
10 #include <net/sock.h>
11 
12 #include "page_pool_priv.h"
13 #include "netdev-genl-gen.h"
14 
15 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);
16 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user.
17  * Ordering: inside rtnl_lock
18  */
19 static DEFINE_MUTEX(page_pools_lock);
20 
21 /* Page pools are only reachable from user space (via netlink) if they are
22  * linked to a netdev at creation time. Following page pool "visibility"
23  * states are possible:
24  *  - normal
25  *    - user.list: linked to real netdev, netdev: real netdev
26  *  - orphaned - real netdev has disappeared
27  *    - user.list: linked to lo, netdev: lo
28  *  - invisible - either (a) created without netdev linking, (b) unlisted due
29  *      to error, or (c) the entire namespace which owned this pool disappeared
30  *    - user.list: unhashed, netdev: unknown
31  */
32 
33 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,
34 			     const struct genl_info *info);
35 
36 static int
37 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill)
38 {
39 	struct page_pool *pool;
40 	struct sk_buff *rsp;
41 	int err;
42 
43 	mutex_lock(&page_pools_lock);
44 	pool = xa_load(&page_pools, id);
45 	if (!pool || hlist_unhashed(&pool->user.list) ||
46 	    !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) {
47 		err = -ENOENT;
48 		goto err_unlock;
49 	}
50 
51 	rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
52 	if (!rsp) {
53 		err = -ENOMEM;
54 		goto err_unlock;
55 	}
56 
57 	err = fill(rsp, pool, info);
58 	if (err)
59 		goto err_free_msg;
60 
61 	mutex_unlock(&page_pools_lock);
62 
63 	return genlmsg_reply(rsp, info);
64 
65 err_free_msg:
66 	nlmsg_free(rsp);
67 err_unlock:
68 	mutex_unlock(&page_pools_lock);
69 	return err;
70 }
71 
72 struct page_pool_dump_cb {
73 	unsigned long ifindex;
74 	u32 pp_id;
75 };
76 
77 static int
78 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
79 			     pp_nl_fill_cb fill)
80 {
81 	struct page_pool_dump_cb *state = (void *)cb->ctx;
82 	const struct genl_info *info = genl_info_dump(cb);
83 	struct net *net = sock_net(skb->sk);
84 	struct net_device *netdev;
85 	struct page_pool *pool;
86 	int err = 0;
87 
88 	rtnl_lock();
89 	mutex_lock(&page_pools_lock);
90 	for_each_netdev_dump(net, netdev, state->ifindex) {
91 		hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
92 			if (state->pp_id && state->pp_id < pool->user.id)
93 				continue;
94 
95 			state->pp_id = pool->user.id;
96 			err = fill(skb, pool, info);
97 			if (err)
98 				goto out;
99 		}
100 
101 		state->pp_id = 0;
102 	}
103 out:
104 	mutex_unlock(&page_pools_lock);
105 	rtnl_unlock();
106 
107 	return err;
108 }
109 
110 static int
111 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,
112 			const struct genl_info *info)
113 {
114 #ifdef CONFIG_PAGE_POOL_STATS
115 	struct page_pool_stats stats = {};
116 	struct nlattr *nest;
117 	void *hdr;
118 
119 	if (!page_pool_get_stats(pool, &stats))
120 		return 0;
121 
122 	hdr = genlmsg_iput(rsp, info);
123 	if (!hdr)
124 		return -EMSGSIZE;
125 
126 	nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO);
127 
128 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) ||
129 	    (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
130 	     nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
131 			 pool->slow.netdev->ifindex)))
132 		goto err_cancel_nest;
133 
134 	nla_nest_end(rsp, nest);
135 
136 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,
137 			 stats.alloc_stats.fast) ||
138 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,
139 			 stats.alloc_stats.slow) ||
140 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,
141 			 stats.alloc_stats.slow_high_order) ||
142 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,
143 			 stats.alloc_stats.empty) ||
144 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,
145 			 stats.alloc_stats.refill) ||
146 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,
147 			 stats.alloc_stats.waive) ||
148 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,
149 			 stats.recycle_stats.cached) ||
150 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,
151 			 stats.recycle_stats.cache_full) ||
152 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,
153 			 stats.recycle_stats.ring) ||
154 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,
155 			 stats.recycle_stats.ring_full) ||
156 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,
157 			 stats.recycle_stats.released_refcnt))
158 		goto err_cancel_msg;
159 
160 	genlmsg_end(rsp, hdr);
161 
162 	return 0;
163 err_cancel_nest:
164 	nla_nest_cancel(rsp, nest);
165 err_cancel_msg:
166 	genlmsg_cancel(rsp, hdr);
167 	return -EMSGSIZE;
168 #else
169 	GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");
170 	return -EOPNOTSUPP;
171 #endif
172 }
173 
174 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
175 				       struct genl_info *info)
176 {
177 	struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];
178 	struct nlattr *nest;
179 	int err;
180 	u32 id;
181 
182 	if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))
183 		return -EINVAL;
184 
185 	nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
186 	err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
187 			       netdev_page_pool_info_nl_policy,
188 			       info->extack);
189 	if (err)
190 		return err;
191 
192 	if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))
193 		return -EINVAL;
194 	if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {
195 		NL_SET_ERR_MSG_ATTR(info->extack,
196 				    tb[NETDEV_A_PAGE_POOL_IFINDEX],
197 				    "selecting by ifindex not supported");
198 		return -EINVAL;
199 	}
200 
201 	id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]);
202 
203 	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);
204 }
205 
206 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
207 					 struct netlink_callback *cb)
208 {
209 	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill);
210 }
211 
212 static int
213 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,
214 		  const struct genl_info *info)
215 {
216 	size_t inflight, refsz;
217 	void *hdr;
218 
219 	hdr = genlmsg_iput(rsp, info);
220 	if (!hdr)
221 		return -EMSGSIZE;
222 
223 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id))
224 		goto err_cancel;
225 
226 	if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&
227 	    nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,
228 			pool->slow.netdev->ifindex))
229 		goto err_cancel;
230 	if (pool->user.napi_id &&
231 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id))
232 		goto err_cancel;
233 
234 	inflight = page_pool_inflight(pool, false);
235 	refsz =	PAGE_SIZE << pool->p.order;
236 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) ||
237 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM,
238 			 inflight * refsz))
239 		goto err_cancel;
240 	if (pool->user.detach_time &&
241 	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME,
242 			 pool->user.detach_time))
243 		goto err_cancel;
244 
245 	genlmsg_end(rsp, hdr);
246 
247 	return 0;
248 err_cancel:
249 	genlmsg_cancel(rsp, hdr);
250 	return -EMSGSIZE;
251 }
252 
253 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd)
254 {
255 	struct genl_info info;
256 	struct sk_buff *ntf;
257 	struct net *net;
258 
259 	lockdep_assert_held(&page_pools_lock);
260 
261 	/* 'invisible' page pools don't matter */
262 	if (hlist_unhashed(&pool->user.list))
263 		return;
264 	net = dev_net(pool->slow.netdev);
265 
266 	if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL))
267 		return;
268 
269 	genl_info_init_ntf(&info, &netdev_nl_family, cmd);
270 
271 	ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
272 	if (!ntf)
273 		return;
274 
275 	if (page_pool_nl_fill(ntf, pool, &info)) {
276 		nlmsg_free(ntf);
277 		return;
278 	}
279 
280 	genlmsg_multicast_netns(&netdev_nl_family, net, ntf,
281 				0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);
282 }
283 
284 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
285 {
286 	u32 id;
287 
288 	if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))
289 		return -EINVAL;
290 
291 	id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]);
292 
293 	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);
294 }
295 
296 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
297 				   struct netlink_callback *cb)
298 {
299 	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill);
300 }
301 
302 int page_pool_list(struct page_pool *pool)
303 {
304 	static u32 id_alloc_next;
305 	int err;
306 
307 	mutex_lock(&page_pools_lock);
308 	err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b,
309 			      &id_alloc_next, GFP_KERNEL);
310 	if (err < 0)
311 		goto err_unlock;
312 
313 	INIT_HLIST_NODE(&pool->user.list);
314 	if (pool->slow.netdev) {
315 		hlist_add_head(&pool->user.list,
316 			       &pool->slow.netdev->page_pools);
317 		pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0;
318 
319 		netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF);
320 	}
321 
322 	mutex_unlock(&page_pools_lock);
323 	return 0;
324 
325 err_unlock:
326 	mutex_unlock(&page_pools_lock);
327 	return err;
328 }
329 
330 void page_pool_detached(struct page_pool *pool)
331 {
332 	mutex_lock(&page_pools_lock);
333 	pool->user.detach_time = ktime_get_boottime_seconds();
334 	netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
335 	mutex_unlock(&page_pools_lock);
336 }
337 
338 void page_pool_unlist(struct page_pool *pool)
339 {
340 	mutex_lock(&page_pools_lock);
341 	netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF);
342 	xa_erase(&page_pools, pool->user.id);
343 	if (!hlist_unhashed(&pool->user.list))
344 		hlist_del(&pool->user.list);
345 	mutex_unlock(&page_pools_lock);
346 }
347 
348 int page_pool_check_memory_provider(struct net_device *dev,
349 				    struct netdev_rx_queue *rxq)
350 {
351 	struct net_devmem_dmabuf_binding *binding = rxq->mp_params.mp_priv;
352 	struct page_pool *pool;
353 	struct hlist_node *n;
354 
355 	if (!binding)
356 		return 0;
357 
358 	mutex_lock(&page_pools_lock);
359 	hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {
360 		if (pool->mp_priv != binding)
361 			continue;
362 
363 		if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) {
364 			mutex_unlock(&page_pools_lock);
365 			return 0;
366 		}
367 	}
368 	mutex_unlock(&page_pools_lock);
369 	return -ENODATA;
370 }
371 
372 static void page_pool_unreg_netdev_wipe(struct net_device *netdev)
373 {
374 	struct page_pool *pool;
375 	struct hlist_node *n;
376 
377 	mutex_lock(&page_pools_lock);
378 	hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {
379 		hlist_del_init(&pool->user.list);
380 		pool->slow.netdev = NET_PTR_POISON;
381 	}
382 	mutex_unlock(&page_pools_lock);
383 }
384 
385 static void page_pool_unreg_netdev(struct net_device *netdev)
386 {
387 	struct page_pool *pool, *last;
388 	struct net_device *lo;
389 
390 	lo = dev_net(netdev)->loopback_dev;
391 
392 	mutex_lock(&page_pools_lock);
393 	last = NULL;
394 	hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
395 		pool->slow.netdev = lo;
396 		netdev_nl_page_pool_event(pool,
397 					  NETDEV_CMD_PAGE_POOL_CHANGE_NTF);
398 		last = pool;
399 	}
400 	if (last)
401 		hlist_splice_init(&netdev->page_pools, &last->user.list,
402 				  &lo->page_pools);
403 	mutex_unlock(&page_pools_lock);
404 }
405 
406 static int
407 page_pool_netdevice_event(struct notifier_block *nb,
408 			  unsigned long event, void *ptr)
409 {
410 	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
411 
412 	if (event != NETDEV_UNREGISTER)
413 		return NOTIFY_DONE;
414 
415 	if (hlist_empty(&netdev->page_pools))
416 		return NOTIFY_OK;
417 
418 	if (netdev->ifindex != LOOPBACK_IFINDEX)
419 		page_pool_unreg_netdev(netdev);
420 	else
421 		page_pool_unreg_netdev_wipe(netdev);
422 	return NOTIFY_OK;
423 }
424 
425 static struct notifier_block page_pool_netdevice_nb = {
426 	.notifier_call = page_pool_netdevice_event,
427 };
428 
429 static int __init page_pool_user_init(void)
430 {
431 	return register_netdevice_notifier(&page_pool_netdevice_nb);
432 }
433 
434 subsys_initcall(page_pool_user_init);
435