1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mutex.h> 4 #include <linux/netdevice.h> 5 #include <linux/xarray.h> 6 #include <net/net_debug.h> 7 #include <net/page_pool/types.h> 8 #include <net/page_pool/helpers.h> 9 #include <net/sock.h> 10 11 #include "page_pool_priv.h" 12 #include "netdev-genl-gen.h" 13 14 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); 15 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user. 16 * Ordering: inside rtnl_lock 17 */ 18 static DEFINE_MUTEX(page_pools_lock); 19 20 /* Page pools are only reachable from user space (via netlink) if they are 21 * linked to a netdev at creation time. Following page pool "visibility" 22 * states are possible: 23 * - normal 24 * - user.list: linked to real netdev, netdev: real netdev 25 * - orphaned - real netdev has disappeared 26 * - user.list: linked to lo, netdev: lo 27 * - invisible - either (a) created without netdev linking, (b) unlisted due 28 * to error, or (c) the entire namespace which owned this pool disappeared 29 * - user.list: unhashed, netdev: unknown 30 */ 31 32 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, 33 const struct genl_info *info); 34 35 static int 36 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill) 37 { 38 struct page_pool *pool; 39 struct sk_buff *rsp; 40 int err; 41 42 mutex_lock(&page_pools_lock); 43 pool = xa_load(&page_pools, id); 44 if (!pool || hlist_unhashed(&pool->user.list) || 45 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { 46 err = -ENOENT; 47 goto err_unlock; 48 } 49 50 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 51 if (!rsp) { 52 err = -ENOMEM; 53 goto err_unlock; 54 } 55 56 err = fill(rsp, pool, info); 57 if (err) 58 goto err_free_msg; 59 60 mutex_unlock(&page_pools_lock); 61 62 return genlmsg_reply(rsp, info); 63 64 err_free_msg: 65 nlmsg_free(rsp); 66 err_unlock: 67 mutex_unlock(&page_pools_lock); 68 return err; 69 } 70 71 struct page_pool_dump_cb { 72 unsigned long ifindex; 73 u32 pp_id; 74 }; 75 76 static int 77 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, 78 pp_nl_fill_cb fill) 79 { 80 struct page_pool_dump_cb *state = (void *)cb->ctx; 81 const struct genl_info *info = genl_info_dump(cb); 82 struct net *net = sock_net(skb->sk); 83 struct net_device *netdev; 84 struct page_pool *pool; 85 int err = 0; 86 87 rtnl_lock(); 88 mutex_lock(&page_pools_lock); 89 for_each_netdev_dump(net, netdev, state->ifindex) { 90 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 91 if (state->pp_id && state->pp_id < pool->user.id) 92 continue; 93 94 state->pp_id = pool->user.id; 95 err = fill(skb, pool, info); 96 if (err) 97 goto out; 98 } 99 100 state->pp_id = 0; 101 } 102 out: 103 mutex_unlock(&page_pools_lock); 104 rtnl_unlock(); 105 106 return err; 107 } 108 109 static int 110 page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, 111 const struct genl_info *info) 112 { 113 #ifdef CONFIG_PAGE_POOL_STATS 114 struct page_pool_stats stats = {}; 115 struct nlattr *nest; 116 void *hdr; 117 118 if (!page_pool_get_stats(pool, &stats)) 119 return 0; 120 121 hdr = genlmsg_iput(rsp, info); 122 if (!hdr) 123 return -EMSGSIZE; 124 125 nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); 126 127 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || 128 (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 129 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 130 pool->slow.netdev->ifindex))) 131 goto err_cancel_nest; 132 133 nla_nest_end(rsp, nest); 134 135 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST, 136 stats.alloc_stats.fast) || 137 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW, 138 stats.alloc_stats.slow) || 139 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER, 140 stats.alloc_stats.slow_high_order) || 141 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY, 142 stats.alloc_stats.empty) || 143 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL, 144 stats.alloc_stats.refill) || 145 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE, 146 stats.alloc_stats.waive) || 147 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED, 148 stats.recycle_stats.cached) || 149 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL, 150 stats.recycle_stats.cache_full) || 151 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING, 152 stats.recycle_stats.ring) || 153 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL, 154 stats.recycle_stats.ring_full) || 155 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT, 156 stats.recycle_stats.released_refcnt)) 157 goto err_cancel_msg; 158 159 genlmsg_end(rsp, hdr); 160 161 return 0; 162 err_cancel_nest: 163 nla_nest_cancel(rsp, nest); 164 err_cancel_msg: 165 genlmsg_cancel(rsp, hdr); 166 return -EMSGSIZE; 167 #else 168 GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS"); 169 return -EOPNOTSUPP; 170 #endif 171 } 172 173 int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, 174 struct genl_info *info) 175 { 176 struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)]; 177 struct nlattr *nest; 178 int err; 179 u32 id; 180 181 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO)) 182 return -EINVAL; 183 184 nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; 185 err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, 186 netdev_page_pool_info_nl_policy, 187 info->extack); 188 if (err) 189 return err; 190 191 if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID)) 192 return -EINVAL; 193 if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) { 194 NL_SET_ERR_MSG_ATTR(info->extack, 195 tb[NETDEV_A_PAGE_POOL_IFINDEX], 196 "selecting by ifindex not supported"); 197 return -EINVAL; 198 } 199 200 id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]); 201 202 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill); 203 } 204 205 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, 206 struct netlink_callback *cb) 207 { 208 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill); 209 } 210 211 static int 212 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, 213 const struct genl_info *info) 214 { 215 size_t inflight, refsz; 216 void *hdr; 217 218 hdr = genlmsg_iput(rsp, info); 219 if (!hdr) 220 return -EMSGSIZE; 221 222 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) 223 goto err_cancel; 224 225 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 226 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 227 pool->slow.netdev->ifindex)) 228 goto err_cancel; 229 if (pool->user.napi_id && 230 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id)) 231 goto err_cancel; 232 233 inflight = page_pool_inflight(pool, false); 234 refsz = PAGE_SIZE << pool->p.order; 235 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || 236 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, 237 inflight * refsz)) 238 goto err_cancel; 239 if (pool->user.detach_time && 240 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, 241 pool->user.detach_time)) 242 goto err_cancel; 243 244 genlmsg_end(rsp, hdr); 245 246 return 0; 247 err_cancel: 248 genlmsg_cancel(rsp, hdr); 249 return -EMSGSIZE; 250 } 251 252 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd) 253 { 254 struct genl_info info; 255 struct sk_buff *ntf; 256 struct net *net; 257 258 lockdep_assert_held(&page_pools_lock); 259 260 /* 'invisible' page pools don't matter */ 261 if (hlist_unhashed(&pool->user.list)) 262 return; 263 net = dev_net(pool->slow.netdev); 264 265 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) 266 return; 267 268 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 269 270 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 271 if (!ntf) 272 return; 273 274 if (page_pool_nl_fill(ntf, pool, &info)) { 275 nlmsg_free(ntf); 276 return; 277 } 278 279 genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 280 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL); 281 } 282 283 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) 284 { 285 u32 id; 286 287 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) 288 return -EINVAL; 289 290 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 291 292 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill); 293 } 294 295 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, 296 struct netlink_callback *cb) 297 { 298 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill); 299 } 300 301 int page_pool_list(struct page_pool *pool) 302 { 303 static u32 id_alloc_next; 304 int err; 305 306 mutex_lock(&page_pools_lock); 307 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, 308 &id_alloc_next, GFP_KERNEL); 309 if (err < 0) 310 goto err_unlock; 311 312 INIT_HLIST_NODE(&pool->user.list); 313 if (pool->slow.netdev) { 314 hlist_add_head(&pool->user.list, 315 &pool->slow.netdev->page_pools); 316 pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0; 317 318 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); 319 } 320 321 mutex_unlock(&page_pools_lock); 322 return 0; 323 324 err_unlock: 325 mutex_unlock(&page_pools_lock); 326 return err; 327 } 328 329 void page_pool_detached(struct page_pool *pool) 330 { 331 mutex_lock(&page_pools_lock); 332 pool->user.detach_time = ktime_get_boottime_seconds(); 333 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 334 mutex_unlock(&page_pools_lock); 335 } 336 337 void page_pool_unlist(struct page_pool *pool) 338 { 339 mutex_lock(&page_pools_lock); 340 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); 341 xa_erase(&page_pools, pool->user.id); 342 if (!hlist_unhashed(&pool->user.list)) 343 hlist_del(&pool->user.list); 344 mutex_unlock(&page_pools_lock); 345 } 346 347 static void page_pool_unreg_netdev_wipe(struct net_device *netdev) 348 { 349 struct page_pool *pool; 350 struct hlist_node *n; 351 352 mutex_lock(&page_pools_lock); 353 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { 354 hlist_del_init(&pool->user.list); 355 pool->slow.netdev = NET_PTR_POISON; 356 } 357 mutex_unlock(&page_pools_lock); 358 } 359 360 static void page_pool_unreg_netdev(struct net_device *netdev) 361 { 362 struct page_pool *pool, *last; 363 struct net_device *lo; 364 365 lo = dev_net(netdev)->loopback_dev; 366 367 mutex_lock(&page_pools_lock); 368 last = NULL; 369 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 370 pool->slow.netdev = lo; 371 netdev_nl_page_pool_event(pool, 372 NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 373 last = pool; 374 } 375 if (last) 376 hlist_splice_init(&netdev->page_pools, &last->user.list, 377 &lo->page_pools); 378 mutex_unlock(&page_pools_lock); 379 } 380 381 static int 382 page_pool_netdevice_event(struct notifier_block *nb, 383 unsigned long event, void *ptr) 384 { 385 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 386 387 if (event != NETDEV_UNREGISTER) 388 return NOTIFY_DONE; 389 390 if (hlist_empty(&netdev->page_pools)) 391 return NOTIFY_OK; 392 393 if (netdev->ifindex != LOOPBACK_IFINDEX) 394 page_pool_unreg_netdev(netdev); 395 else 396 page_pool_unreg_netdev_wipe(netdev); 397 return NOTIFY_OK; 398 } 399 400 static struct notifier_block page_pool_netdevice_nb = { 401 .notifier_call = page_pool_netdevice_event, 402 }; 403 404 static int __init page_pool_user_init(void) 405 { 406 return register_netdevice_notifier(&page_pool_netdevice_nb); 407 } 408 409 subsys_initcall(page_pool_user_init); 410