1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mutex.h> 4 #include <linux/netdevice.h> 5 #include <linux/xarray.h> 6 #include <net/net_debug.h> 7 #include <net/netdev_rx_queue.h> 8 #include <net/page_pool/helpers.h> 9 #include <net/page_pool/types.h> 10 #include <net/sock.h> 11 12 #include "devmem.h" 13 #include "page_pool_priv.h" 14 #include "netdev-genl-gen.h" 15 16 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); 17 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user. 18 * Ordering: inside rtnl_lock 19 */ 20 static DEFINE_MUTEX(page_pools_lock); 21 22 /* Page pools are only reachable from user space (via netlink) if they are 23 * linked to a netdev at creation time. Following page pool "visibility" 24 * states are possible: 25 * - normal 26 * - user.list: linked to real netdev, netdev: real netdev 27 * - orphaned - real netdev has disappeared 28 * - user.list: linked to lo, netdev: lo 29 * - invisible - either (a) created without netdev linking, (b) unlisted due 30 * to error, or (c) the entire namespace which owned this pool disappeared 31 * - user.list: unhashed, netdev: unknown 32 */ 33 34 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, 35 const struct genl_info *info); 36 37 static int 38 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill) 39 { 40 struct page_pool *pool; 41 struct sk_buff *rsp; 42 int err; 43 44 mutex_lock(&page_pools_lock); 45 pool = xa_load(&page_pools, id); 46 if (!pool || hlist_unhashed(&pool->user.list) || 47 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { 48 err = -ENOENT; 49 goto err_unlock; 50 } 51 52 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 53 if (!rsp) { 54 err = -ENOMEM; 55 goto err_unlock; 56 } 57 58 err = fill(rsp, pool, info); 59 if (err) 60 goto err_free_msg; 61 62 mutex_unlock(&page_pools_lock); 63 64 return genlmsg_reply(rsp, info); 65 66 err_free_msg: 67 nlmsg_free(rsp); 68 err_unlock: 69 mutex_unlock(&page_pools_lock); 70 return err; 71 } 72 73 struct page_pool_dump_cb { 74 unsigned long ifindex; 75 u32 pp_id; 76 }; 77 78 static int 79 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, 80 pp_nl_fill_cb fill) 81 { 82 struct page_pool_dump_cb *state = (void *)cb->ctx; 83 const struct genl_info *info = genl_info_dump(cb); 84 struct net *net = sock_net(skb->sk); 85 struct net_device *netdev; 86 struct page_pool *pool; 87 int err = 0; 88 89 rtnl_lock(); 90 mutex_lock(&page_pools_lock); 91 for_each_netdev_dump(net, netdev, state->ifindex) { 92 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 93 if (state->pp_id && state->pp_id < pool->user.id) 94 continue; 95 96 state->pp_id = pool->user.id; 97 err = fill(skb, pool, info); 98 if (err) 99 goto out; 100 } 101 102 state->pp_id = 0; 103 } 104 out: 105 mutex_unlock(&page_pools_lock); 106 rtnl_unlock(); 107 108 return err; 109 } 110 111 static int 112 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, 113 const struct genl_info *info) 114 { 115 #ifdef CONFIG_PAGE_POOL_STATS 116 struct page_pool_stats stats = {}; 117 struct nlattr *nest; 118 void *hdr; 119 120 if (!page_pool_get_stats(pool, &stats)) 121 return 0; 122 123 hdr = genlmsg_iput(rsp, info); 124 if (!hdr) 125 return -EMSGSIZE; 126 127 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); 128 129 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || 130 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 131 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 132 pool->slow.netdev->ifindex))) 133 goto err_cancel_nest; 134 135 nla_nest_end(rsp, nest); 136 137 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST, 138 stats.alloc_stats.fast) || 139 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, 140 stats.alloc_stats.slow) || 141 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, 142 stats.alloc_stats.slow_high_order) || 143 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, 144 stats.alloc_stats.empty) || 145 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, 146 stats.alloc_stats.refill) || 147 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, 148 stats.alloc_stats.waive) || 149 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, 150 stats.recycle_stats.cached) || 151 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, 152 stats.recycle_stats.cache_full) || 153 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, 154 stats.recycle_stats.ring) || 155 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, 156 stats.recycle_stats.ring_full) || 157 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, 158 stats.recycle_stats.released_refcnt)) 159 goto err_cancel_msg; 160 161 genlmsg_end(rsp, hdr); 162 163 return 0; 164 err_cancel_nest: 165 nla_nest_cancel(rsp, nest); 166 err_cancel_msg: 167 genlmsg_cancel(rsp, hdr); 168 return -EMSGSIZE; 169 #else 170 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS"); 171 return -EOPNOTSUPP; 172 #endif 173 } 174 175 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, 176 struct genl_info *info) 177 { 178 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)]; 179 struct nlattr *nest; 180 int err; 181 u32 id; 182 183 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO)) 184 return -EINVAL; 185 186 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; 187 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, 188 netdev_page_pool_info_nl_policy, 189 info->extack); 190 if (err) 191 return err; 192 193 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID)) 194 return -EINVAL; 195 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) { 196 NL_SET_ERR_MSG_ATTR(info->extack, 197 tb[NETDEV_A_PAGE_POOL_IFINDEX], 198 "selecting by ifindex not supported"); 199 return -EINVAL; 200 } 201 202 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]); 203 204 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill); 205 } 206 207 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, 208 struct netlink_callback *cb) 209 { 210 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill); 211 } 212 213 static int 214 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, 215 const struct genl_info *info) 216 { 217 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 218 size_t inflight, refsz; 219 void *hdr; 220 221 hdr = genlmsg_iput(rsp, info); 222 if (!hdr) 223 return -EMSGSIZE; 224 225 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) 226 goto err_cancel; 227 228 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 229 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 230 pool->slow.netdev->ifindex)) 231 goto err_cancel; 232 if (pool->user.napi_id && 233 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id)) 234 goto err_cancel; 235 236 inflight = page_pool_inflight(pool, false); 237 refsz = PAGE_SIZE << pool->p.order; 238 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || 239 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, 240 inflight * refsz)) 241 goto err_cancel; 242 if (pool->user.detach_time && 243 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, 244 pool->user.detach_time)) 245 goto err_cancel; 246 247 if (binding && nla_put_u32(rsp, NETDEV_A_PAGE_POOL_DMABUF, binding->id)) 248 goto err_cancel; 249 250 genlmsg_end(rsp, hdr); 251 252 return 0; 253 err_cancel: 254 genlmsg_cancel(rsp, hdr); 255 return -EMSGSIZE; 256 } 257 258 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd) 259 { 260 struct genl_info info; 261 struct sk_buff *ntf; 262 struct net *net; 263 264 lockdep_assert_held(&page_pools_lock); 265 266 /* 'invisible' page pools don't matter */ 267 if (hlist_unhashed(&pool->user.list)) 268 return; 269 net = dev_net(pool->slow.netdev); 270 271 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) 272 return; 273 274 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 275 276 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 277 if (!ntf) 278 return; 279 280 if (page_pool_nl_fill(ntf, pool, &info)) { 281 nlmsg_free(ntf); 282 return; 283 } 284 285 genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 286 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL); 287 } 288 289 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) 290 { 291 u32 id; 292 293 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) 294 return -EINVAL; 295 296 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 297 298 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill); 299 } 300 301 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, 302 struct netlink_callback *cb) 303 { 304 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill); 305 } 306 307 int page_pool_list(struct page_pool *pool) 308 { 309 static u32 id_alloc_next; 310 int err; 311 312 mutex_lock(&page_pools_lock); 313 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, 314 &id_alloc_next, GFP_KERNEL); 315 if (err < 0) 316 goto err_unlock; 317 318 INIT_HLIST_NODE(&pool->user.list); 319 if (pool->slow.netdev) { 320 hlist_add_head(&pool->user.list, 321 &pool->slow.netdev->page_pools); 322 pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0; 323 324 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); 325 } 326 327 mutex_unlock(&page_pools_lock); 328 return 0; 329 330 err_unlock: 331 mutex_unlock(&page_pools_lock); 332 return err; 333 } 334 335 void page_pool_detached(struct page_pool *pool) 336 { 337 mutex_lock(&page_pools_lock); 338 pool->user.detach_time = ktime_get_boottime_seconds(); 339 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 340 mutex_unlock(&page_pools_lock); 341 } 342 343 void page_pool_unlist(struct page_pool *pool) 344 { 345 mutex_lock(&page_pools_lock); 346 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); 347 xa_erase(&page_pools, pool->user.id); 348 if (!hlist_unhashed(&pool->user.list)) 349 hlist_del(&pool->user.list); 350 mutex_unlock(&page_pools_lock); 351 } 352 353 int page_pool_check_memory_provider(struct net_device *dev, 354 struct netdev_rx_queue *rxq) 355 { 356 struct net_devmem_dmabuf_binding *binding = rxq->mp_params.mp_priv; 357 struct page_pool *pool; 358 struct hlist_node *n; 359 360 if (!binding) 361 return 0; 362 363 mutex_lock(&page_pools_lock); 364 hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) { 365 if (pool->mp_priv != binding) 366 continue; 367 368 if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) { 369 mutex_unlock(&page_pools_lock); 370 return 0; 371 } 372 } 373 mutex_unlock(&page_pools_lock); 374 return -ENODATA; 375 } 376 377 static void page_pool_unreg_netdev_wipe(struct net_device *netdev) 378 { 379 struct page_pool *pool; 380 struct hlist_node *n; 381 382 mutex_lock(&page_pools_lock); 383 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { 384 hlist_del_init(&pool->user.list); 385 pool->slow.netdev = NET_PTR_POISON; 386 } 387 mutex_unlock(&page_pools_lock); 388 } 389 390 static void page_pool_unreg_netdev(struct net_device *netdev) 391 { 392 struct page_pool *pool, *last; 393 struct net_device *lo; 394 395 lo = dev_net(netdev)->loopback_dev; 396 397 mutex_lock(&page_pools_lock); 398 last = NULL; 399 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 400 pool->slow.netdev = lo; 401 netdev_nl_page_pool_event(pool, 402 NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 403 last = pool; 404 } 405 if (last) 406 hlist_splice_init(&netdev->page_pools, &last->user.list, 407 &lo->page_pools); 408 mutex_unlock(&page_pools_lock); 409 } 410 411 static int 412 page_pool_netdevice_event(struct notifier_block *nb, 413 unsigned long event, void *ptr) 414 { 415 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 416 417 if (event != NETDEV_UNREGISTER) 418 return NOTIFY_DONE; 419 420 if (hlist_empty(&netdev->page_pools)) 421 return NOTIFY_OK; 422 423 if (netdev->ifindex != LOOPBACK_IFINDEX) 424 page_pool_unreg_netdev(netdev); 425 else 426 page_pool_unreg_netdev_wipe(netdev); 427 return NOTIFY_OK; 428 } 429 430 static struct notifier_block page_pool_netdevice_nb = { 431 .notifier_call = page_pool_netdevice_event, 432 }; 433 434 static int __init page_pool_user_init(void) 435 { 436 return register_netdevice_notifier(&page_pool_netdevice_nb); 437 } 438 439 subsys_initcall(page_pool_user_init); 440