1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mutex.h> 4 #include <linux/netdevice.h> 5 #include <linux/xarray.h> 6 #include <net/net_debug.h> 7 #include <net/page_pool/types.h> 8 #include <net/sock.h> 9 10 #include "page_pool_priv.h" 11 #include "netdev-genl-gen.h" 12 13 static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1); 14 /* Protects: page_pools, netdevice->page_pools, pool->slow.netdev, pool->user. 15 * Ordering: inside rtnl_lock 16 */ 17 static DEFINE_MUTEX(page_pools_lock); 18 19 /* Page pools are only reachable from user space (via netlink) if they are 20 * linked to a netdev at creation time. Following page pool "visibility" 21 * states are possible: 22 * - normal 23 * - user.list: linked to real netdev, netdev: real netdev 24 * - orphaned - real netdev has disappeared 25 * - user.list: linked to lo, netdev: lo 26 * - invisible - either (a) created without netdev linking, (b) unlisted due 27 * to error, or (c) the entire namespace which owned this pool disappeared 28 * - user.list: unhashed, netdev: unknown 29 */ 30 31 typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool, 32 const struct genl_info *info); 33 34 static int 35 netdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill) 36 { 37 struct page_pool *pool; 38 struct sk_buff *rsp; 39 int err; 40 41 mutex_lock(&page_pools_lock); 42 pool = xa_load(&page_pools, id); 43 if (!pool || hlist_unhashed(&pool->user.list) || 44 !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) { 45 err = -ENOENT; 46 goto err_unlock; 47 } 48 49 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 50 if (!rsp) { 51 err = -ENOMEM; 52 goto err_unlock; 53 } 54 55 err = fill(rsp, pool, info); 56 if (err) 57 goto err_free_msg; 58 59 mutex_unlock(&page_pools_lock); 60 61 return genlmsg_reply(rsp, info); 62 63 err_free_msg: 64 nlmsg_free(rsp); 65 err_unlock: 66 mutex_unlock(&page_pools_lock); 67 return err; 68 } 69 70 struct page_pool_dump_cb { 71 unsigned long ifindex; 72 u32 pp_id; 73 }; 74 75 static int 76 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, 77 pp_nl_fill_cb fill) 78 { 79 struct page_pool_dump_cb *state = (void *)cb->ctx; 80 const struct genl_info *info = genl_info_dump(cb); 81 struct net *net = sock_net(skb->sk); 82 struct net_device *netdev; 83 struct page_pool *pool; 84 int err = 0; 85 86 rtnl_lock(); 87 mutex_lock(&page_pools_lock); 88 for_each_netdev_dump(net, netdev, state->ifindex) { 89 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 90 if (state->pp_id && state->pp_id < pool->user.id) 91 continue; 92 93 state->pp_id = pool->user.id; 94 err = fill(skb, pool, info); 95 if (err) 96 break; 97 } 98 99 state->pp_id = 0; 100 } 101 mutex_unlock(&page_pools_lock); 102 rtnl_unlock(); 103 104 if (skb->len && err == -EMSGSIZE) 105 return skb->len; 106 return err; 107 } 108 109 static int 110 page_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool, 111 const struct genl_info *info) 112 { 113 size_t inflight, refsz; 114 void *hdr; 115 116 hdr = genlmsg_iput(rsp, info); 117 if (!hdr) 118 return -EMSGSIZE; 119 120 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id)) 121 goto err_cancel; 122 123 if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && 124 nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX, 125 pool->slow.netdev->ifindex)) 126 goto err_cancel; 127 if (pool->user.napi_id && 128 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, pool->user.napi_id)) 129 goto err_cancel; 130 131 inflight = page_pool_inflight(pool, false); 132 refsz = PAGE_SIZE << pool->p.order; 133 if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) || 134 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM, 135 inflight * refsz)) 136 goto err_cancel; 137 if (pool->user.detach_time && 138 nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME, 139 pool->user.detach_time)) 140 goto err_cancel; 141 142 genlmsg_end(rsp, hdr); 143 144 return 0; 145 err_cancel: 146 genlmsg_cancel(rsp, hdr); 147 return -EMSGSIZE; 148 } 149 150 static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd) 151 { 152 struct genl_info info; 153 struct sk_buff *ntf; 154 struct net *net; 155 156 lockdep_assert_held(&page_pools_lock); 157 158 /* 'invisible' page pools don't matter */ 159 if (hlist_unhashed(&pool->user.list)) 160 return; 161 net = dev_net(pool->slow.netdev); 162 163 if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL)) 164 return; 165 166 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 167 168 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 169 if (!ntf) 170 return; 171 172 if (page_pool_nl_fill(ntf, pool, &info)) { 173 nlmsg_free(ntf); 174 return; 175 } 176 177 genlmsg_multicast_netns(&netdev_nl_family, net, ntf, 178 0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL); 179 } 180 181 int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) 182 { 183 u32 id; 184 185 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID)) 186 return -EINVAL; 187 188 id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 189 190 return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill); 191 } 192 193 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, 194 struct netlink_callback *cb) 195 { 196 return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill); 197 } 198 199 int page_pool_list(struct page_pool *pool) 200 { 201 static u32 id_alloc_next; 202 int err; 203 204 mutex_lock(&page_pools_lock); 205 err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b, 206 &id_alloc_next, GFP_KERNEL); 207 if (err < 0) 208 goto err_unlock; 209 210 if (pool->slow.netdev) { 211 hlist_add_head(&pool->user.list, 212 &pool->slow.netdev->page_pools); 213 pool->user.napi_id = pool->p.napi ? pool->p.napi->napi_id : 0; 214 215 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF); 216 } 217 218 mutex_unlock(&page_pools_lock); 219 return 0; 220 221 err_unlock: 222 mutex_unlock(&page_pools_lock); 223 return err; 224 } 225 226 void page_pool_detached(struct page_pool *pool) 227 { 228 mutex_lock(&page_pools_lock); 229 pool->user.detach_time = ktime_get_boottime_seconds(); 230 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 231 mutex_unlock(&page_pools_lock); 232 } 233 234 void page_pool_unlist(struct page_pool *pool) 235 { 236 mutex_lock(&page_pools_lock); 237 netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF); 238 xa_erase(&page_pools, pool->user.id); 239 hlist_del(&pool->user.list); 240 mutex_unlock(&page_pools_lock); 241 } 242 243 static void page_pool_unreg_netdev_wipe(struct net_device *netdev) 244 { 245 struct page_pool *pool; 246 struct hlist_node *n; 247 248 mutex_lock(&page_pools_lock); 249 hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) { 250 hlist_del_init(&pool->user.list); 251 pool->slow.netdev = NET_PTR_POISON; 252 } 253 mutex_unlock(&page_pools_lock); 254 } 255 256 static void page_pool_unreg_netdev(struct net_device *netdev) 257 { 258 struct page_pool *pool, *last; 259 struct net_device *lo; 260 261 lo = dev_net(netdev)->loopback_dev; 262 263 mutex_lock(&page_pools_lock); 264 last = NULL; 265 hlist_for_each_entry(pool, &netdev->page_pools, user.list) { 266 pool->slow.netdev = lo; 267 netdev_nl_page_pool_event(pool, 268 NETDEV_CMD_PAGE_POOL_CHANGE_NTF); 269 last = pool; 270 } 271 if (last) 272 hlist_splice_init(&netdev->page_pools, &last->user.list, 273 &lo->page_pools); 274 mutex_unlock(&page_pools_lock); 275 } 276 277 static int 278 page_pool_netdevice_event(struct notifier_block *nb, 279 unsigned long event, void *ptr) 280 { 281 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 282 283 if (event != NETDEV_UNREGISTER) 284 return NOTIFY_DONE; 285 286 if (hlist_empty(&netdev->page_pools)) 287 return NOTIFY_OK; 288 289 if (netdev->ifindex != LOOPBACK_IFINDEX) 290 page_pool_unreg_netdev(netdev); 291 else 292 page_pool_unreg_netdev_wipe(netdev); 293 return NOTIFY_OK; 294 } 295 296 static struct notifier_block page_pool_netdevice_nb = { 297 .notifier_call = page_pool_netdevice_event, 298 }; 299 300 static int __init page_pool_user_init(void) 301 { 302 return register_netdevice_notifier(&page_pool_netdevice_nb); 303 } 304 305 subsys_initcall(page_pool_user_init); 306