1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mutex.h> 4 #include <linux/netdevice.h> 5 #include <linux/xarray.h> 6 #include <net/net_debug.h> 7 #include <net/netdev_rx_queue.h> 8 #include <net/page_pool/helpers.h> 9 #include <net/page_pool/types.h> 10 #include <net/sock.h> 11 12 #include "page_pool_priv.h" 13 #include "netdev-genl-gen.h" 14 15 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); 16 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user. 17 * Ordering: inside rtnl_lock 18 */ 19 static DEFINE_MUTEX(page_pools_lock); 20 21 /* Page pools are only reachable from user space (via netlink) if they are 22 * linked to a netdev at creation time. Following page pool "visibility" 23 * states are possible: 24 * - normal 25 * - user.list: linked to real netdev, netdev: real netdev 26 * - orphaned - real netdev has disappeared 27 * - user.list: linked to lo, netdev: lo 28 * - invisible - either (a) created without netdev linking, (b) unlisted due 29 * to error, or (c) the entire namespace which owned this pool disappeared 30 * - user.list: unhashed, netdev: unknown 31 */ 32 33 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, 34 const struct genl_info *info); 35 36 static int 37 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill) 38 { 39 struct page_pool *pool; 40 struct sk_buff *rsp; 41 int err; 42 43 mutex_lock(&page_pools_lock); 44 pool = xa_load(&page_pools, id); 45 if (!pool || hlist_unhashed(&pool->user.list) || 46 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { 47 err = -ENOENT; 48 goto err_unlock; 49 } 50 51 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 52 if (!rsp) { 53 err = -ENOMEM; 54 goto err_unlock; 55 } 56 57 err = fill(rsp, pool, info); 58 if (err) 59 goto err_free_msg; 60 61 mutex_unlock(&page_pools_lock); 62 63 return genlmsg_reply(rsp, info); 64 65 err_free_msg: 66 nlmsg_free(rsp); 67 err_unlock: 68 mutex_unlock(&page_pools_lock); 69 return err; 70 } 71 72 struct page_pool_dump_cb { 73 unsigned long ifindex; 74 u32 pp_id; 75 }; 76 77 static int 78 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, 79 pp_nl_fill_cb fill) 80 { 81 struct page_pool_dump_cb *state = (void *)cb->ctx; 82 const struct genl_info *info = genl_info_dump(cb); 83 struct net *net = sock_net(skb->sk); 84 struct net_device *netdev; 85 struct page_pool *pool; 86 int err = 0; 87 88 rtnl_lock(); 89 mutex_lock(&page_pools_lock); 90 for_each_netdev_dump(net, netdev, state->ifindex) { 91 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 92 if (state->pp_id && state->pp_id < pool->user.id) 93 continue; 94 95 state->pp_id = pool->user.id; 96 err = fill(skb, pool, info); 97 if (err) 98 goto out; 99 } 100 101 state->pp_id = 0; 102 } 103 out: 104 mutex_unlock(&page_pools_lock); 105 rtnl_unlock(); 106 107 return err; 108 } 109 110 static int 111 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, 112 const struct genl_info *info) 113 { 114 #ifdef CONFIG_PAGE_POOL_STATS 115 struct page_pool_stats stats = {}; 116 struct nlattr *nest; 117 void *hdr; 118 119 if (!page_pool_get_stats(pool, &stats)) 120 return 0; 121 122 hdr = genlmsg_iput(rsp, info); 123 if (!hdr) 124 return -EMSGSIZE; 125 126 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); 127 128 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || 129 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 130 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 131 pool->slow.netdev->ifindex))) 132 goto err_cancel_nest; 133 134 nla_nest_end(rsp, nest); 135 136 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST, 137 stats.alloc_stats.fast) || 138 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, 139 stats.alloc_stats.slow) || 140 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, 141 stats.alloc_stats.slow_high_order) || 142 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, 143 stats.alloc_stats.empty) || 144 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, 145 stats.alloc_stats.refill) || 146 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, 147 stats.alloc_stats.waive) || 148 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, 149 stats.recycle_stats.cached) || 150 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, 151 stats.recycle_stats.cache_full) || 152 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, 153 stats.recycle_stats.ring) || 154 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, 155 stats.recycle_stats.ring_full) || 156 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, 157 stats.recycle_stats.released_refcnt)) 158 goto err_cancel_msg; 159 160 genlmsg_end(rsp, hdr); 161 162 return 0; 163 err_cancel_nest: 164 nla_nest_cancel(rsp, nest); 165 err_cancel_msg: 166 genlmsg_cancel(rsp, hdr); 167 return -EMSGSIZE; 168 #else 169 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS"); 170 return -EOPNOTSUPP; 171 #endif 172 } 173 174 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, 175 struct genl_info *info) 176 { 177 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)]; 178 struct nlattr *nest; 179 int err; 180 u32 id; 181 182 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO)) 183 return -EINVAL; 184 185 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; 186 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, 187 netdev_page_pool_info_nl_policy, 188 info->extack); 189 if (err) 190 return err; 191 192 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID)) 193 return -EINVAL; 194 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) { 195 NL_SET_ERR_MSG_ATTR(info->extack, 196 tb[NETDEV_A_PAGE_POOL_IFINDEX], 197 "selecting by ifindex not supported"); 198 return -EINVAL; 199 } 200 201 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]); 202 203 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill); 204 } 205 206 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, 207 struct netlink_callback *cb) 208 { 209 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill); 210 } 211 212 static int 213 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, 214 const struct genl_info *info) 215 { 216 size_t inflight, refsz; 217 void *hdr; 218 219 hdr = genlmsg_iput(rsp, info); 220 if (!hdr) 221 return -EMSGSIZE; 222 223 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) 224 goto err_cancel; 225 226 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 227 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 228 pool->slow.netdev->ifindex)) 229 goto err_cancel; 230 if (pool->user.napi_id && 231 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id)) 232 goto err_cancel; 233 234 inflight = page_pool_inflight(pool, false); 235 refsz = PAGE_SIZE << pool->p.order; 236 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || 237 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, 238 inflight * refsz)) 239 goto err_cancel; 240 if (pool->user.detach_time && 241 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, 242 pool->user.detach_time)) 243 goto err_cancel; 244 245 genlmsg_end(rsp, hdr); 246 247 return 0; 248 err_cancel: 249 genlmsg_cancel(rsp, hdr); 250 return -EMSGSIZE; 251 } 252 253 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd) 254 { 255 struct genl_info info; 256 struct sk_buff *ntf; 257 struct net *net; 258 259 lockdep_assert_held(&page_pools_lock); 260 261 /* 'invisible' page pools don't matter */ 262 if (hlist_unhashed(&pool->user.list)) 263 return; 264 net = dev_net(pool->slow.netdev); 265 266 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) 267 return; 268 269 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 270 271 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 272 if (!ntf) 273 return; 274 275 if (page_pool_nl_fill(ntf, pool, &info)) { 276 nlmsg_free(ntf); 277 return; 278 } 279 280 genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 281 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL); 282 } 283 284 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) 285 { 286 u32 id; 287 288 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) 289 return -EINVAL; 290 291 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 292 293 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill); 294 } 295 296 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, 297 struct netlink_callback *cb) 298 { 299 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill); 300 } 301 302 int page_pool_list(struct page_pool *pool) 303 { 304 static u32 id_alloc_next; 305 int err; 306 307 mutex_lock(&page_pools_lock); 308 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, 309 &id_alloc_next, GFP_KERNEL); 310 if (err < 0) 311 goto err_unlock; 312 313 INIT_HLIST_NODE(&pool->user.list); 314 if (pool->slow.netdev) { 315 hlist_add_head(&pool->user.list, 316 &pool->slow.netdev->page_pools); 317 pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0; 318 319 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); 320 } 321 322 mutex_unlock(&page_pools_lock); 323 return 0; 324 325 err_unlock: 326 mutex_unlock(&page_pools_lock); 327 return err; 328 } 329 330 void page_pool_detached(struct page_pool *pool) 331 { 332 mutex_lock(&page_pools_lock); 333 pool->user.detach_time = ktime_get_boottime_seconds(); 334 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 335 mutex_unlock(&page_pools_lock); 336 } 337 338 void page_pool_unlist(struct page_pool *pool) 339 { 340 mutex_lock(&page_pools_lock); 341 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); 342 xa_erase(&page_pools, pool->user.id); 343 if (!hlist_unhashed(&pool->user.list)) 344 hlist_del(&pool->user.list); 345 mutex_unlock(&page_pools_lock); 346 } 347 348 int page_pool_check_memory_provider(struct net_device *dev, 349 struct netdev_rx_queue *rxq) 350 { 351 struct net_devmem_dmabuf_binding *binding = rxq->mp_params.mp_priv; 352 struct page_pool *pool; 353 struct hlist_node *n; 354 355 if (!binding) 356 return 0; 357 358 mutex_lock(&page_pools_lock); 359 hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) { 360 if (pool->mp_priv != binding) 361 continue; 362 363 if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) { 364 mutex_unlock(&page_pools_lock); 365 return 0; 366 } 367 } 368 mutex_unlock(&page_pools_lock); 369 return -ENODATA; 370 } 371 372 static void page_pool_unreg_netdev_wipe(struct net_device *netdev) 373 { 374 struct page_pool *pool; 375 struct hlist_node *n; 376 377 mutex_lock(&page_pools_lock); 378 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { 379 hlist_del_init(&pool->user.list); 380 pool->slow.netdev = NET_PTR_POISON; 381 } 382 mutex_unlock(&page_pools_lock); 383 } 384 385 static void page_pool_unreg_netdev(struct net_device *netdev) 386 { 387 struct page_pool *pool, *last; 388 struct net_device *lo; 389 390 lo = dev_net(netdev)->loopback_dev; 391 392 mutex_lock(&page_pools_lock); 393 last = NULL; 394 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 395 pool->slow.netdev = lo; 396 netdev_nl_page_pool_event(pool, 397 NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 398 last = pool; 399 } 400 if (last) 401 hlist_splice_init(&netdev->page_pools, &last->user.list, 402 &lo->page_pools); 403 mutex_unlock(&page_pools_lock); 404 } 405 406 static int 407 page_pool_netdevice_event(struct notifier_block *nb, 408 unsigned long event, void *ptr) 409 { 410 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 411 412 if (event != NETDEV_UNREGISTER) 413 return NOTIFY_DONE; 414 415 if (hlist_empty(&netdev->page_pools)) 416 return NOTIFY_OK; 417 418 if (netdev->ifindex != LOOPBACK_IFINDEX) 419 page_pool_unreg_netdev(netdev); 420 else 421 page_pool_unreg_netdev_wipe(netdev); 422 return NOTIFY_OK; 423 } 424 425 static struct notifier_block page_pool_netdevice_nb = { 426 .notifier_call = page_pool_netdevice_event, 427 }; 428 429 static int __init page_pool_user_init(void) 430 { 431 return register_netdevice_notifier(&page_pool_netdevice_nb); 432 } 433 434 subsys_initcall(page_pool_user_init); 435