1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mutex.h> 4 #include <linux/netdevice.h> 5 #include <linux/xarray.h> 6 #include <net/busy_poll.h> 7 #include <net/net_debug.h> 8 #include <net/netdev_rx_queue.h> 9 #include <net/page_pool/helpers.h> 10 #include <net/page_pool/types.h> 11 #include <net/page_pool/memory_provider.h> 12 #include <net/sock.h> 13 14 #include "page_pool_priv.h" 15 #include "netdev-genl-gen.h" 16 17 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); 18 /* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev, 19 * pool->user. 20 * Ordering: inside rtnl_lock 21 */ 22 DEFINE_MUTEX(page_pools_lock); 23 24 /* Page pools are only reachable from user space (via netlink) if they are 25 * linked to a netdev at creation time. Following page pool "visibility" 26 * states are possible: 27 * - normal 28 * - user.list: linked to real netdev, netdev: real netdev 29 * - orphaned - real netdev has disappeared 30 * - user.list: linked to lo, netdev: lo 31 * - invisible - either (a) created without netdev linking, (b) unlisted due 32 * to error, or (c) the entire namespace which owned this pool disappeared 33 * - user.list: unhashed, netdev: unknown 34 */ 35 36 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, 37 const struct genl_info *info); 38 39 static int 40 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill) 41 { 42 struct page_pool *pool; 43 struct sk_buff *rsp; 44 int err; 45 46 mutex_lock(&page_pools_lock); 47 pool = xa_load(&page_pools, id); 48 if (!pool || hlist_unhashed(&pool->user.list) || 49 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { 50 err = -ENOENT; 51 goto err_unlock; 52 } 53 54 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 55 if (!rsp) { 56 err = -ENOMEM; 57 goto err_unlock; 58 } 59 60 err = fill(rsp, pool, info); 61 if (err) 62 goto err_free_msg; 63 64 mutex_unlock(&page_pools_lock); 65 66 return genlmsg_reply(rsp, info); 67 68 err_free_msg: 69 nlmsg_free(rsp); 70 err_unlock: 71 mutex_unlock(&page_pools_lock); 72 return err; 73 } 74 75 struct page_pool_dump_cb { 76 unsigned long ifindex; 77 u32 pp_id; 78 }; 79 80 static int 81 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, 82 pp_nl_fill_cb fill, struct nlattr *ifindex_attr) 83 { 84 struct page_pool_dump_cb *state = (void *)cb->ctx; 85 const struct genl_info *info = genl_info_dump(cb); 86 struct net *net = sock_net(skb->sk); 87 struct net_device *netdev; 88 struct page_pool *pool; 89 int err = 0; 90 91 if (ifindex_attr) 92 state->ifindex = nla_get_u32(ifindex_attr); 93 94 rtnl_lock(); 95 mutex_lock(&page_pools_lock); 96 for_each_netdev_dump(net, netdev, state->ifindex) { 97 /* Either the provided ifindex doesn't exist or done dumping */ 98 if (ifindex_attr && 99 netdev->ifindex != nla_get_u32(ifindex_attr)) 100 break; 101 102 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 103 if (state->pp_id && state->pp_id < pool->user.id) 104 continue; 105 106 state->pp_id = pool->user.id; 107 err = fill(skb, pool, info); 108 if (err) 109 goto out; 110 } 111 112 state->pp_id = 0; 113 } 114 out: 115 mutex_unlock(&page_pools_lock); 116 rtnl_unlock(); 117 118 return err; 119 } 120 121 static int 122 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, 123 const struct genl_info *info) 124 { 125 #ifdef CONFIG_PAGE_POOL_STATS 126 struct page_pool_stats stats = {}; 127 struct nlattr *nest; 128 void *hdr; 129 130 if (!page_pool_get_stats(pool, &stats)) 131 return 0; 132 133 hdr = genlmsg_iput(rsp, info); 134 if (!hdr) 135 return -EMSGSIZE; 136 137 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); 138 if (!nest) 139 goto err_cancel_msg; 140 141 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || 142 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 143 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 144 pool->slow.netdev->ifindex))) 145 goto err_cancel_nest; 146 147 nla_nest_end(rsp, nest); 148 149 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST, 150 stats.alloc_stats.fast) || 151 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, 152 stats.alloc_stats.slow) || 153 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, 154 stats.alloc_stats.slow_high_order) || 155 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, 156 stats.alloc_stats.empty) || 157 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, 158 stats.alloc_stats.refill) || 159 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, 160 stats.alloc_stats.waive) || 161 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, 162 stats.recycle_stats.cached) || 163 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, 164 stats.recycle_stats.cache_full) || 165 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, 166 stats.recycle_stats.ring) || 167 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, 168 stats.recycle_stats.ring_full) || 169 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, 170 stats.recycle_stats.released_refcnt)) 171 goto err_cancel_msg; 172 173 genlmsg_end(rsp, hdr); 174 175 return 0; 176 err_cancel_nest: 177 nla_nest_cancel(rsp, nest); 178 err_cancel_msg: 179 genlmsg_cancel(rsp, hdr); 180 return -EMSGSIZE; 181 #else 182 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS"); 183 return -EOPNOTSUPP; 184 #endif 185 } 186 187 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, 188 struct genl_info *info) 189 { 190 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)]; 191 struct nlattr *nest; 192 int err; 193 u32 id; 194 195 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO)) 196 return -EINVAL; 197 198 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; 199 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, 200 netdev_page_pool_info_nl_policy, 201 info->extack); 202 if (err) 203 return err; 204 205 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID)) 206 return -EINVAL; 207 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) { 208 NL_SET_ERR_MSG_ATTR(info->extack, 209 tb[NETDEV_A_PAGE_POOL_IFINDEX], 210 "selecting by ifindex not supported"); 211 return -EINVAL; 212 } 213 214 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]); 215 216 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill); 217 } 218 219 static const struct netlink_range_validation page_pool_ifindex_range = { 220 .min = 1ULL, 221 .max = S32_MAX, 222 }; 223 224 static const struct nla_policy 225 page_pool_stat_info_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = { 226 [NETDEV_A_PAGE_POOL_IFINDEX] = 227 NLA_POLICY_FULL_RANGE(NLA_U32, &page_pool_ifindex_range), 228 }; 229 230 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, 231 struct netlink_callback *cb) 232 { 233 struct nlattr *tb[ARRAY_SIZE(page_pool_stat_info_policy)]; 234 const struct genl_info *info = genl_info_dump(cb); 235 struct nlattr *ifindex_attr = NULL; 236 237 if (info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]) { 238 struct nlattr *nest; 239 int err; 240 241 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; 242 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, 243 page_pool_stat_info_policy, 244 info->extack); 245 if (err) 246 return err; 247 248 ifindex_attr = tb[NETDEV_A_PAGE_POOL_IFINDEX]; 249 } 250 251 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill, 252 ifindex_attr); 253 } 254 255 static int 256 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, 257 const struct genl_info *info) 258 { 259 size_t inflight, refsz; 260 unsigned int napi_id; 261 void *hdr; 262 263 hdr = genlmsg_iput(rsp, info); 264 if (!hdr) 265 return -EMSGSIZE; 266 267 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) 268 goto err_cancel; 269 270 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 271 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 272 pool->slow.netdev->ifindex)) 273 goto err_cancel; 274 275 napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0; 276 if (napi_id_valid(napi_id) && 277 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, napi_id)) 278 goto err_cancel; 279 280 inflight = page_pool_inflight(pool, false); 281 refsz = PAGE_SIZE << pool->p.order; 282 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || 283 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, 284 inflight * refsz)) 285 goto err_cancel; 286 if (pool->user.detach_time && 287 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, 288 ktime_divns(pool->user.detach_time, NSEC_PER_SEC))) 289 goto err_cancel; 290 291 if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL)) 292 goto err_cancel; 293 294 genlmsg_end(rsp, hdr); 295 296 return 0; 297 err_cancel: 298 genlmsg_cancel(rsp, hdr); 299 return -EMSGSIZE; 300 } 301 302 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd) 303 { 304 struct genl_info info; 305 struct sk_buff *ntf; 306 struct net *net; 307 308 lockdep_assert_held(&page_pools_lock); 309 310 /* 'invisible' page pools don't matter */ 311 if (hlist_unhashed(&pool->user.list)) 312 return; 313 net = dev_net(pool->slow.netdev); 314 315 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) 316 return; 317 318 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 319 320 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 321 if (!ntf) 322 return; 323 324 if (page_pool_nl_fill(ntf, pool, &info)) { 325 nlmsg_free(ntf); 326 return; 327 } 328 329 genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 330 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL); 331 } 332 333 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) 334 { 335 u32 id; 336 337 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) 338 return -EINVAL; 339 340 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 341 342 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill); 343 } 344 345 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, 346 struct netlink_callback *cb) 347 { 348 const struct genl_info *info = genl_info_dump(cb); 349 350 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill, 351 info->attrs[NETDEV_A_PAGE_POOL_IFINDEX]); 352 } 353 354 int page_pool_list(struct page_pool *pool) 355 { 356 static u32 id_alloc_next; 357 int err; 358 359 mutex_lock(&page_pools_lock); 360 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, 361 &id_alloc_next, GFP_KERNEL); 362 if (err < 0) 363 goto err_unlock; 364 365 INIT_HLIST_NODE(&pool->user.list); 366 if (pool->slow.netdev) { 367 hlist_add_head(&pool->user.list, 368 &pool->slow.netdev->page_pools); 369 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); 370 } 371 372 mutex_unlock(&page_pools_lock); 373 return 0; 374 375 err_unlock: 376 mutex_unlock(&page_pools_lock); 377 return err; 378 } 379 380 void page_pool_detached(struct page_pool *pool) 381 { 382 mutex_lock(&page_pools_lock); 383 pool->user.detach_time = ktime_get_boottime(); 384 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 385 mutex_unlock(&page_pools_lock); 386 } 387 388 void page_pool_unlist(struct page_pool *pool) 389 { 390 mutex_lock(&page_pools_lock); 391 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); 392 xa_erase(&page_pools, pool->user.id); 393 if (!hlist_unhashed(&pool->user.list)) 394 hlist_del(&pool->user.list); 395 mutex_unlock(&page_pools_lock); 396 } 397 398 int page_pool_check_memory_provider(struct net_device *dev, 399 struct netdev_rx_queue *rxq) 400 { 401 void *binding = rxq->mp_params.mp_priv; 402 struct page_pool *pool; 403 struct hlist_node *n; 404 405 if (!binding) 406 return 0; 407 408 mutex_lock(&page_pools_lock); 409 hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) { 410 if (pool->mp_priv != binding) 411 continue; 412 413 if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) { 414 mutex_unlock(&page_pools_lock); 415 return 0; 416 } 417 } 418 mutex_unlock(&page_pools_lock); 419 return -ENODATA; 420 } 421 422 static void page_pool_unreg_netdev_wipe(struct net_device *netdev) 423 { 424 struct page_pool *pool; 425 struct hlist_node *n; 426 427 mutex_lock(&page_pools_lock); 428 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { 429 hlist_del_init(&pool->user.list); 430 pool->slow.netdev = NET_PTR_POISON; 431 } 432 mutex_unlock(&page_pools_lock); 433 } 434 435 static void page_pool_unreg_netdev(struct net_device *netdev) 436 { 437 struct page_pool *pool, *last; 438 struct net_device *lo; 439 440 lo = dev_net(netdev)->loopback_dev; 441 442 mutex_lock(&page_pools_lock); 443 last = NULL; 444 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 445 pool->slow.netdev = lo; 446 netdev_nl_page_pool_event(pool, 447 NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 448 last = pool; 449 } 450 if (last) 451 hlist_splice_init(&netdev->page_pools, &last->user.list, 452 &lo->page_pools); 453 mutex_unlock(&page_pools_lock); 454 } 455 456 static int 457 page_pool_netdevice_event(struct notifier_block *nb, 458 unsigned long event, void *ptr) 459 { 460 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 461 462 if (event != NETDEV_UNREGISTER) 463 return NOTIFY_DONE; 464 465 if (hlist_empty(&netdev->page_pools)) 466 return NOTIFY_OK; 467 468 if (netdev->ifindex != LOOPBACK_IFINDEX) 469 page_pool_unreg_netdev(netdev); 470 else 471 page_pool_unreg_netdev_wipe(netdev); 472 return NOTIFY_OK; 473 } 474 475 static struct notifier_block page_pool_netdevice_nb = { 476 .notifier_call = page_pool_netdevice_event, 477 }; 478 479 static int __init page_pool_user_init(void) 480 { 481 return register_netdevice_notifier(&page_pool_netdevice_nb); 482 } 483 484 subsys_initcall(page_pool_user_init); 485