1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mutex.h> 4 #include <linux/netdevice.h> 5 #include <linux/xarray.h> 6 #include <net/busy_poll.h> 7 #include <net/net_debug.h> 8 #include <net/netdev_rx_queue.h> 9 #include <net/page_pool/helpers.h> 10 #include <net/page_pool/types.h> 11 #include <net/sock.h> 12 13 #include "devmem.h" 14 #include "page_pool_priv.h" 15 #include "netdev-genl-gen.h" 16 17 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); 18 /* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev, 19 * pool->user. 20 * Ordering: inside rtnl_lock 21 */ 22 DEFINE_MUTEX(page_pools_lock); 23 24 /* Page pools are only reachable from user space (via netlink) if they are 25 * linked to a netdev at creation time. Following page pool "visibility" 26 * states are possible: 27 * - normal 28 * - user.list: linked to real netdev, netdev: real netdev 29 * - orphaned - real netdev has disappeared 30 * - user.list: linked to lo, netdev: lo 31 * - invisible - either (a) created without netdev linking, (b) unlisted due 32 * to error, or (c) the entire namespace which owned this pool disappeared 33 * - user.list: unhashed, netdev: unknown 34 */ 35 36 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, 37 const struct genl_info *info); 38 39 static int 40 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill) 41 { 42 struct page_pool *pool; 43 struct sk_buff *rsp; 44 int err; 45 46 mutex_lock(&page_pools_lock); 47 pool = xa_load(&page_pools, id); 48 if (!pool || hlist_unhashed(&pool->user.list) || 49 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { 50 err = -ENOENT; 51 goto err_unlock; 52 } 53 54 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 55 if (!rsp) { 56 err = -ENOMEM; 57 goto err_unlock; 58 } 59 60 err = fill(rsp, pool, info); 61 if (err) 62 goto err_free_msg; 63 64 mutex_unlock(&page_pools_lock); 65 66 return genlmsg_reply(rsp, info); 67 68 err_free_msg: 69 nlmsg_free(rsp); 70 err_unlock: 71 mutex_unlock(&page_pools_lock); 72 return err; 73 } 74 75 struct page_pool_dump_cb { 76 unsigned long ifindex; 77 u32 pp_id; 78 }; 79 80 static int 81 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, 82 pp_nl_fill_cb fill) 83 { 84 struct page_pool_dump_cb *state = (void *)cb->ctx; 85 const struct genl_info *info = genl_info_dump(cb); 86 struct net *net = sock_net(skb->sk); 87 struct net_device *netdev; 88 struct page_pool *pool; 89 int err = 0; 90 91 rtnl_lock(); 92 mutex_lock(&page_pools_lock); 93 for_each_netdev_dump(net, netdev, state->ifindex) { 94 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 95 if (state->pp_id && state->pp_id < pool->user.id) 96 continue; 97 98 state->pp_id = pool->user.id; 99 err = fill(skb, pool, info); 100 if (err) 101 goto out; 102 } 103 104 state->pp_id = 0; 105 } 106 out: 107 mutex_unlock(&page_pools_lock); 108 rtnl_unlock(); 109 110 return err; 111 } 112 113 static int 114 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, 115 const struct genl_info *info) 116 { 117 #ifdef CONFIG_PAGE_POOL_STATS 118 struct page_pool_stats stats = {}; 119 struct nlattr *nest; 120 void *hdr; 121 122 if (!page_pool_get_stats(pool, &stats)) 123 return 0; 124 125 hdr = genlmsg_iput(rsp, info); 126 if (!hdr) 127 return -EMSGSIZE; 128 129 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); 130 131 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || 132 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 133 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 134 pool->slow.netdev->ifindex))) 135 goto err_cancel_nest; 136 137 nla_nest_end(rsp, nest); 138 139 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST, 140 stats.alloc_stats.fast) || 141 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, 142 stats.alloc_stats.slow) || 143 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, 144 stats.alloc_stats.slow_high_order) || 145 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, 146 stats.alloc_stats.empty) || 147 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, 148 stats.alloc_stats.refill) || 149 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, 150 stats.alloc_stats.waive) || 151 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, 152 stats.recycle_stats.cached) || 153 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, 154 stats.recycle_stats.cache_full) || 155 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, 156 stats.recycle_stats.ring) || 157 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, 158 stats.recycle_stats.ring_full) || 159 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, 160 stats.recycle_stats.released_refcnt)) 161 goto err_cancel_msg; 162 163 genlmsg_end(rsp, hdr); 164 165 return 0; 166 err_cancel_nest: 167 nla_nest_cancel(rsp, nest); 168 err_cancel_msg: 169 genlmsg_cancel(rsp, hdr); 170 return -EMSGSIZE; 171 #else 172 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS"); 173 return -EOPNOTSUPP; 174 #endif 175 } 176 177 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, 178 struct genl_info *info) 179 { 180 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)]; 181 struct nlattr *nest; 182 int err; 183 u32 id; 184 185 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO)) 186 return -EINVAL; 187 188 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; 189 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, 190 netdev_page_pool_info_nl_policy, 191 info->extack); 192 if (err) 193 return err; 194 195 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID)) 196 return -EINVAL; 197 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) { 198 NL_SET_ERR_MSG_ATTR(info->extack, 199 tb[NETDEV_A_PAGE_POOL_IFINDEX], 200 "selecting by ifindex not supported"); 201 return -EINVAL; 202 } 203 204 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]); 205 206 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill); 207 } 208 209 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, 210 struct netlink_callback *cb) 211 { 212 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill); 213 } 214 215 static int 216 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, 217 const struct genl_info *info) 218 { 219 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 220 size_t inflight, refsz; 221 unsigned int napi_id; 222 void *hdr; 223 224 hdr = genlmsg_iput(rsp, info); 225 if (!hdr) 226 return -EMSGSIZE; 227 228 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) 229 goto err_cancel; 230 231 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 232 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 233 pool->slow.netdev->ifindex)) 234 goto err_cancel; 235 236 napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0; 237 if (napi_id >= MIN_NAPI_ID && 238 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, napi_id)) 239 goto err_cancel; 240 241 inflight = page_pool_inflight(pool, false); 242 refsz = PAGE_SIZE << pool->p.order; 243 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || 244 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, 245 inflight * refsz)) 246 goto err_cancel; 247 if (pool->user.detach_time && 248 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, 249 pool->user.detach_time)) 250 goto err_cancel; 251 252 if (binding && nla_put_u32(rsp, NETDEV_A_PAGE_POOL_DMABUF, binding->id)) 253 goto err_cancel; 254 255 genlmsg_end(rsp, hdr); 256 257 return 0; 258 err_cancel: 259 genlmsg_cancel(rsp, hdr); 260 return -EMSGSIZE; 261 } 262 263 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd) 264 { 265 struct genl_info info; 266 struct sk_buff *ntf; 267 struct net *net; 268 269 lockdep_assert_held(&page_pools_lock); 270 271 /* 'invisible' page pools don't matter */ 272 if (hlist_unhashed(&pool->user.list)) 273 return; 274 net = dev_net(pool->slow.netdev); 275 276 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) 277 return; 278 279 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 280 281 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 282 if (!ntf) 283 return; 284 285 if (page_pool_nl_fill(ntf, pool, &info)) { 286 nlmsg_free(ntf); 287 return; 288 } 289 290 genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 291 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL); 292 } 293 294 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) 295 { 296 u32 id; 297 298 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) 299 return -EINVAL; 300 301 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 302 303 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill); 304 } 305 306 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, 307 struct netlink_callback *cb) 308 { 309 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill); 310 } 311 312 int page_pool_list(struct page_pool *pool) 313 { 314 static u32 id_alloc_next; 315 int err; 316 317 mutex_lock(&page_pools_lock); 318 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, 319 &id_alloc_next, GFP_KERNEL); 320 if (err < 0) 321 goto err_unlock; 322 323 INIT_HLIST_NODE(&pool->user.list); 324 if (pool->slow.netdev) { 325 hlist_add_head(&pool->user.list, 326 &pool->slow.netdev->page_pools); 327 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); 328 } 329 330 mutex_unlock(&page_pools_lock); 331 return 0; 332 333 err_unlock: 334 mutex_unlock(&page_pools_lock); 335 return err; 336 } 337 338 void page_pool_detached(struct page_pool *pool) 339 { 340 mutex_lock(&page_pools_lock); 341 pool->user.detach_time = ktime_get_boottime_seconds(); 342 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 343 mutex_unlock(&page_pools_lock); 344 } 345 346 void page_pool_unlist(struct page_pool *pool) 347 { 348 mutex_lock(&page_pools_lock); 349 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); 350 xa_erase(&page_pools, pool->user.id); 351 if (!hlist_unhashed(&pool->user.list)) 352 hlist_del(&pool->user.list); 353 mutex_unlock(&page_pools_lock); 354 } 355 356 int page_pool_check_memory_provider(struct net_device *dev, 357 struct netdev_rx_queue *rxq) 358 { 359 struct net_devmem_dmabuf_binding *binding = rxq->mp_params.mp_priv; 360 struct page_pool *pool; 361 struct hlist_node *n; 362 363 if (!binding) 364 return 0; 365 366 mutex_lock(&page_pools_lock); 367 hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) { 368 if (pool->mp_priv != binding) 369 continue; 370 371 if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) { 372 mutex_unlock(&page_pools_lock); 373 return 0; 374 } 375 } 376 mutex_unlock(&page_pools_lock); 377 return -ENODATA; 378 } 379 380 static void page_pool_unreg_netdev_wipe(struct net_device *netdev) 381 { 382 struct page_pool *pool; 383 struct hlist_node *n; 384 385 mutex_lock(&page_pools_lock); 386 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { 387 hlist_del_init(&pool->user.list); 388 pool->slow.netdev = NET_PTR_POISON; 389 } 390 mutex_unlock(&page_pools_lock); 391 } 392 393 static void page_pool_unreg_netdev(struct net_device *netdev) 394 { 395 struct page_pool *pool, *last; 396 struct net_device *lo; 397 398 lo = dev_net(netdev)->loopback_dev; 399 400 mutex_lock(&page_pools_lock); 401 last = NULL; 402 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 403 pool->slow.netdev = lo; 404 netdev_nl_page_pool_event(pool, 405 NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 406 last = pool; 407 } 408 if (last) 409 hlist_splice_init(&netdev->page_pools, &last->user.list, 410 &lo->page_pools); 411 mutex_unlock(&page_pools_lock); 412 } 413 414 static int 415 page_pool_netdevice_event(struct notifier_block *nb, 416 unsigned long event, void *ptr) 417 { 418 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 419 420 if (event != NETDEV_UNREGISTER) 421 return NOTIFY_DONE; 422 423 if (hlist_empty(&netdev->page_pools)) 424 return NOTIFY_OK; 425 426 if (netdev->ifindex != LOOPBACK_IFINDEX) 427 page_pool_unreg_netdev(netdev); 428 else 429 page_pool_unreg_netdev_wipe(netdev); 430 return NOTIFY_OK; 431 } 432 433 static struct notifier_block page_pool_netdevice_nb = { 434 .notifier_call = page_pool_netdevice_event, 435 }; 436 437 static int __init page_pool_user_init(void) 438 { 439 return register_netdevice_notifier(&page_pool_netdevice_nb); 440 } 441 442 subsys_initcall(page_pool_user_init); 443