1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/pid_namespace.h>
6 #include <linux/rtnetlink.h>
7 #include <net/busy_poll.h>
8 #include <net/net_namespace.h>
9 #include <net/netdev_queues.h>
10 #include <net/netdev_rx_queue.h>
11 #include <net/sock.h>
12 #include <net/xdp.h>
13 #include <net/xdp_sock.h>
14 #include <net/page_pool/memory_provider.h>
15
16 #include "dev.h"
17 #include "devmem.h"
18 #include "netdev-genl-gen.h"
19
20 struct netdev_nl_dump_ctx {
21 unsigned long ifindex;
22 unsigned int rxq_idx;
23 unsigned int txq_idx;
24 unsigned int napi_id;
25 };
26
netdev_dump_ctx(struct netlink_callback * cb)27 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
28 {
29 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
30
31 return (struct netdev_nl_dump_ctx *)cb->ctx;
32 }
33
34 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)35 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
36 const struct genl_info *info)
37 {
38 u64 xsk_features = 0;
39 u64 xdp_rx_meta = 0;
40 void *hdr;
41
42 netdev_assert_locked(netdev); /* note: rtnl_lock may not be held! */
43
44 hdr = genlmsg_iput(rsp, info);
45 if (!hdr)
46 return -EMSGSIZE;
47
48 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
49 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
50 xdp_rx_meta |= flag;
51 XDP_METADATA_KFUNC_xxx
52 #undef XDP_METADATA_KFUNC
53
54 if (netdev->xsk_tx_metadata_ops) {
55 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
56 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
57 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
58 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
59 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time)
60 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO;
61 }
62
63 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
64 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
65 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
66 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
67 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
68 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
69 xsk_features, NETDEV_A_DEV_PAD))
70 goto err_cancel_msg;
71
72 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
73 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
74 netdev->xdp_zc_max_segs))
75 goto err_cancel_msg;
76 }
77
78 genlmsg_end(rsp, hdr);
79
80 return 0;
81
82 err_cancel_msg:
83 genlmsg_cancel(rsp, hdr);
84 return -EMSGSIZE;
85 }
86
87 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)88 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
89 {
90 struct genl_info info;
91 struct sk_buff *ntf;
92
93 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
94 NETDEV_NLGRP_MGMT))
95 return;
96
97 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
98
99 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
100 if (!ntf)
101 return;
102
103 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
104 nlmsg_free(ntf);
105 return;
106 }
107
108 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
109 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
110 }
111
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)112 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
113 {
114 struct net_device *netdev;
115 struct sk_buff *rsp;
116 u32 ifindex;
117 int err;
118
119 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
120 return -EINVAL;
121
122 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
123
124 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
125 if (!rsp)
126 return -ENOMEM;
127
128 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
129 if (!netdev) {
130 err = -ENODEV;
131 goto err_free_msg;
132 }
133
134 err = netdev_nl_dev_fill(netdev, rsp, info);
135 netdev_unlock(netdev);
136
137 if (err)
138 goto err_free_msg;
139
140 return genlmsg_reply(rsp, info);
141
142 err_free_msg:
143 nlmsg_free(rsp);
144 return err;
145 }
146
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)147 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
148 {
149 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
150 struct net *net = sock_net(skb->sk);
151 int err;
152
153 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
155 if (err < 0)
156 return err;
157 }
158
159 return 0;
160 }
161
162 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)163 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
164 const struct genl_info *info)
165 {
166 unsigned long irq_suspend_timeout;
167 unsigned long gro_flush_timeout;
168 u32 napi_defer_hard_irqs;
169 void *hdr;
170 pid_t pid;
171
172 if (!napi->dev->up)
173 return 0;
174
175 hdr = genlmsg_iput(rsp, info);
176 if (!hdr)
177 return -EMSGSIZE;
178
179 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
180 goto nla_put_failure;
181
182 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
183 goto nla_put_failure;
184
185 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
186 goto nla_put_failure;
187
188 if (nla_put_uint(rsp, NETDEV_A_NAPI_THREADED,
189 napi_get_threaded(napi)))
190 goto nla_put_failure;
191
192 if (napi->thread) {
193 pid = task_pid_nr_ns(napi->thread,
194 task_active_pid_ns(current));
195 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
196 goto nla_put_failure;
197 }
198
199 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
200 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
201 napi_defer_hard_irqs))
202 goto nla_put_failure;
203
204 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
205 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
206 irq_suspend_timeout))
207 goto nla_put_failure;
208
209 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
210 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
211 gro_flush_timeout))
212 goto nla_put_failure;
213
214 genlmsg_end(rsp, hdr);
215
216 return 0;
217
218 nla_put_failure:
219 genlmsg_cancel(rsp, hdr);
220 return -EMSGSIZE;
221 }
222
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)223 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
224 {
225 struct napi_struct *napi;
226 struct sk_buff *rsp;
227 u32 napi_id;
228 int err;
229
230 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
231 return -EINVAL;
232
233 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
234
235 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
236 if (!rsp)
237 return -ENOMEM;
238
239 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
240 if (napi) {
241 err = netdev_nl_napi_fill_one(rsp, napi, info);
242 netdev_unlock(napi->dev);
243 } else {
244 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
245 err = -ENOENT;
246 }
247
248 if (err) {
249 goto err_free_msg;
250 } else if (!rsp->len) {
251 err = -ENOENT;
252 goto err_free_msg;
253 }
254
255 return genlmsg_reply(rsp, info);
256
257 err_free_msg:
258 nlmsg_free(rsp);
259 return err;
260 }
261
262 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)263 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
264 const struct genl_info *info,
265 struct netdev_nl_dump_ctx *ctx)
266 {
267 struct napi_struct *napi;
268 unsigned int prev_id;
269 int err = 0;
270
271 if (!netdev->up)
272 return err;
273
274 prev_id = UINT_MAX;
275 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
276 if (!napi_id_valid(napi->napi_id))
277 continue;
278
279 /* Dump continuation below depends on the list being sorted */
280 WARN_ON_ONCE(napi->napi_id >= prev_id);
281 prev_id = napi->napi_id;
282
283 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
284 continue;
285
286 err = netdev_nl_napi_fill_one(rsp, napi, info);
287 if (err)
288 return err;
289 ctx->napi_id = napi->napi_id;
290 }
291 return err;
292 }
293
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)294 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
295 {
296 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
297 const struct genl_info *info = genl_info_dump(cb);
298 struct net *net = sock_net(skb->sk);
299 struct net_device *netdev;
300 u32 ifindex = 0;
301 int err = 0;
302
303 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
304 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
305
306 if (ifindex) {
307 netdev = netdev_get_by_index_lock(net, ifindex);
308 if (netdev) {
309 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
310 netdev_unlock(netdev);
311 } else {
312 err = -ENODEV;
313 }
314 } else {
315 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
316 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
317 if (err < 0)
318 break;
319 ctx->napi_id = 0;
320 }
321 }
322
323 return err;
324 }
325
326 static int
netdev_nl_napi_set_config(struct napi_struct * napi,struct genl_info * info)327 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
328 {
329 u64 irq_suspend_timeout = 0;
330 u64 gro_flush_timeout = 0;
331 u8 threaded = 0;
332 u32 defer = 0;
333
334 if (info->attrs[NETDEV_A_NAPI_THREADED]) {
335 int ret;
336
337 threaded = nla_get_uint(info->attrs[NETDEV_A_NAPI_THREADED]);
338 ret = napi_set_threaded(napi, threaded);
339 if (ret)
340 return ret;
341 }
342
343 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
344 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
345 napi_set_defer_hard_irqs(napi, defer);
346 }
347
348 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
349 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
350 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
351 }
352
353 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
354 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
355 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
356 }
357
358 return 0;
359 }
360
netdev_nl_napi_set_doit(struct sk_buff * skb,struct genl_info * info)361 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
362 {
363 struct napi_struct *napi;
364 unsigned int napi_id;
365 int err;
366
367 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
368 return -EINVAL;
369
370 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
371
372 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
373 if (napi) {
374 err = netdev_nl_napi_set_config(napi, info);
375 netdev_unlock(napi->dev);
376 } else {
377 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
378 err = -ENOENT;
379 }
380
381 return err;
382 }
383
nla_put_napi_id(struct sk_buff * skb,const struct napi_struct * napi)384 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi)
385 {
386 if (napi && napi_id_valid(napi->napi_id))
387 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id);
388 return 0;
389 }
390
391 static int
netdev_nl_queue_fill_lease(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type)392 netdev_nl_queue_fill_lease(struct sk_buff *rsp, struct net_device *netdev,
393 u32 q_idx, u32 q_type)
394 {
395 struct net_device *orig_netdev = netdev;
396 struct nlattr *nest_lease, *nest_queue;
397 struct netdev_rx_queue *rxq;
398 struct net *net, *peer_net;
399
400 rxq = __netif_get_rx_queue_lease(&netdev, &q_idx, NETIF_PHYS_TO_VIRT);
401 if (!rxq || orig_netdev == netdev)
402 return 0;
403
404 nest_lease = nla_nest_start(rsp, NETDEV_A_QUEUE_LEASE);
405 if (!nest_lease)
406 goto nla_put_failure;
407
408 nest_queue = nla_nest_start(rsp, NETDEV_A_LEASE_QUEUE);
409 if (!nest_queue)
410 goto nla_put_failure;
411 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx))
412 goto nla_put_failure;
413 if (nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type))
414 goto nla_put_failure;
415 nla_nest_end(rsp, nest_queue);
416
417 if (nla_put_u32(rsp, NETDEV_A_LEASE_IFINDEX,
418 READ_ONCE(netdev->ifindex)))
419 goto nla_put_failure;
420
421 rcu_read_lock();
422 peer_net = dev_net_rcu(netdev);
423 net = dev_net_rcu(orig_netdev);
424 if (!net_eq(net, peer_net)) {
425 s32 id = peernet2id_alloc(net, peer_net, GFP_ATOMIC);
426
427 if (nla_put_s32(rsp, NETDEV_A_LEASE_NETNS_ID, id))
428 goto nla_put_failure_unlock;
429 }
430 rcu_read_unlock();
431 nla_nest_end(rsp, nest_lease);
432 return 0;
433
434 nla_put_failure_unlock:
435 rcu_read_unlock();
436 nla_put_failure:
437 return -ENOMEM;
438 }
439
440 static int
__netdev_nl_queue_fill_mp(struct sk_buff * rsp,struct netdev_rx_queue * rxq)441 __netdev_nl_queue_fill_mp(struct sk_buff *rsp, struct netdev_rx_queue *rxq)
442 {
443 struct pp_memory_provider_params *params = &rxq->mp_params;
444
445 if (params->mp_ops &&
446 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq))
447 return -EMSGSIZE;
448
449 #ifdef CONFIG_XDP_SOCKETS
450 if (rxq->pool)
451 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
452 return -EMSGSIZE;
453 #endif
454 return 0;
455 }
456
457 static int
netdev_nl_queue_fill_mp(struct sk_buff * rsp,struct net_device * netdev,struct netdev_rx_queue * rxq)458 netdev_nl_queue_fill_mp(struct sk_buff *rsp, struct net_device *netdev,
459 struct netdev_rx_queue *rxq)
460 {
461 struct netdev_rx_queue *hw_rxq;
462 int ret;
463
464 hw_rxq = rxq->lease;
465 if (!hw_rxq || !netif_is_queue_leasee(netdev))
466 return __netdev_nl_queue_fill_mp(rsp, rxq);
467
468 netdev_lock(hw_rxq->dev);
469 ret = __netdev_nl_queue_fill_mp(rsp, hw_rxq);
470 netdev_unlock(hw_rxq->dev);
471 return ret;
472 }
473
474 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)475 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
476 u32 q_idx, u32 q_type, const struct genl_info *info)
477 {
478 struct netdev_rx_queue *rxq;
479 struct netdev_queue *txq;
480 void *hdr;
481
482 hdr = genlmsg_iput(rsp, info);
483 if (!hdr)
484 return -EMSGSIZE;
485
486 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
487 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
488 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
489 goto nla_put_failure;
490
491 switch (q_type) {
492 case NETDEV_QUEUE_TYPE_RX:
493 rxq = __netif_get_rx_queue(netdev, q_idx);
494 if (nla_put_napi_id(rsp, rxq->napi))
495 goto nla_put_failure;
496 if (netdev_nl_queue_fill_lease(rsp, netdev, q_idx, q_type))
497 goto nla_put_failure;
498 if (netdev_nl_queue_fill_mp(rsp, netdev, rxq))
499 goto nla_put_failure;
500 break;
501 case NETDEV_QUEUE_TYPE_TX:
502 txq = netdev_get_tx_queue(netdev, q_idx);
503 if (nla_put_napi_id(rsp, txq->napi))
504 goto nla_put_failure;
505 #ifdef CONFIG_XDP_SOCKETS
506 if (txq->pool)
507 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
508 goto nla_put_failure;
509 #endif
510 break;
511 }
512
513 genlmsg_end(rsp, hdr);
514
515 return 0;
516
517 nla_put_failure:
518 genlmsg_cancel(rsp, hdr);
519 return -EMSGSIZE;
520 }
521
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)522 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
523 u32 q_type)
524 {
525 switch (q_type) {
526 case NETDEV_QUEUE_TYPE_RX:
527 if (q_id >= netdev->real_num_rx_queues)
528 return -EINVAL;
529 return 0;
530 case NETDEV_QUEUE_TYPE_TX:
531 if (q_id >= netdev->real_num_tx_queues)
532 return -EINVAL;
533 }
534 return 0;
535 }
536
537 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)538 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
539 u32 q_type, const struct genl_info *info)
540 {
541 int err;
542
543 if (!netdev->up)
544 return -ENOENT;
545
546 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
547 if (err)
548 return err;
549
550 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
551 }
552
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)553 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
554 {
555 u32 q_id, q_type, ifindex;
556 struct net_device *netdev;
557 struct sk_buff *rsp;
558 int err;
559
560 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
561 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
562 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
563 return -EINVAL;
564
565 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
566 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
567 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
568
569 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
570 if (!rsp)
571 return -ENOMEM;
572
573 netdev = netdev_get_by_index_lock_ops_compat(genl_info_net(info),
574 ifindex);
575 if (netdev) {
576 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
577 netdev_unlock_ops_compat(netdev);
578 } else {
579 err = -ENODEV;
580 }
581
582 if (err)
583 goto err_free_msg;
584
585 return genlmsg_reply(rsp, info);
586
587 err_free_msg:
588 nlmsg_free(rsp);
589 return err;
590 }
591
592 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)593 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
594 const struct genl_info *info,
595 struct netdev_nl_dump_ctx *ctx)
596 {
597 int err = 0;
598
599 if (!netdev->up)
600 return err;
601
602 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
603 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
604 NETDEV_QUEUE_TYPE_RX, info);
605 if (err)
606 return err;
607 }
608 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
609 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
610 NETDEV_QUEUE_TYPE_TX, info);
611 if (err)
612 return err;
613 }
614
615 return err;
616 }
617
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)618 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
619 {
620 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
621 const struct genl_info *info = genl_info_dump(cb);
622 struct net *net = sock_net(skb->sk);
623 struct net_device *netdev;
624 u32 ifindex = 0;
625 int err = 0;
626
627 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
628 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
629
630 if (ifindex) {
631 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex);
632 if (netdev) {
633 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
634 netdev_unlock_ops_compat(netdev);
635 } else {
636 err = -ENODEV;
637 }
638 } else {
639 for_each_netdev_lock_ops_compat_scoped(net, netdev,
640 ctx->ifindex) {
641 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
642 if (err < 0)
643 break;
644 ctx->rxq_idx = 0;
645 ctx->txq_idx = 0;
646 }
647 }
648
649 return err;
650 }
651
652 #define NETDEV_STAT_NOT_SET (~0ULL)
653
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)654 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
655 {
656 const u64 *add = _add;
657 u64 *sum = _sum;
658
659 while (size) {
660 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
661 *sum += *add;
662 sum++;
663 add++;
664 size -= 8;
665 }
666 }
667
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)668 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
669 {
670 if (value == NETDEV_STAT_NOT_SET)
671 return 0;
672 return nla_put_uint(rsp, attr_id, value);
673 }
674
675 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)676 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
677 {
678 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
679 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
680 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
681 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
682 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
683 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) ||
684 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
685 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
686 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
687 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
688 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
689 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
690 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
691 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
692 return -EMSGSIZE;
693 return 0;
694 }
695
696 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)697 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
698 {
699 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
700 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
701 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
702 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
703 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
704 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
705 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
706 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
707 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
708 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
709 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
710 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
711 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
712 return -EMSGSIZE;
713 return 0;
714 }
715
716 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)717 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
718 u32 q_type, int i, const struct genl_info *info)
719 {
720 const struct netdev_stat_ops *ops = netdev->stat_ops;
721 struct netdev_queue_stats_rx rx;
722 struct netdev_queue_stats_tx tx;
723 void *hdr;
724
725 hdr = genlmsg_iput(rsp, info);
726 if (!hdr)
727 return -EMSGSIZE;
728 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
729 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
730 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
731 goto nla_put_failure;
732
733 switch (q_type) {
734 case NETDEV_QUEUE_TYPE_RX:
735 memset(&rx, 0xff, sizeof(rx));
736 ops->get_queue_stats_rx(netdev, i, &rx);
737 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
738 goto nla_cancel;
739 if (netdev_nl_stats_write_rx(rsp, &rx))
740 goto nla_put_failure;
741 break;
742 case NETDEV_QUEUE_TYPE_TX:
743 memset(&tx, 0xff, sizeof(tx));
744 ops->get_queue_stats_tx(netdev, i, &tx);
745 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
746 goto nla_cancel;
747 if (netdev_nl_stats_write_tx(rsp, &tx))
748 goto nla_put_failure;
749 break;
750 }
751
752 genlmsg_end(rsp, hdr);
753 return 0;
754
755 nla_cancel:
756 genlmsg_cancel(rsp, hdr);
757 return 0;
758 nla_put_failure:
759 genlmsg_cancel(rsp, hdr);
760 return -EMSGSIZE;
761 }
762
763 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)764 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
765 const struct genl_info *info,
766 struct netdev_nl_dump_ctx *ctx)
767 {
768 const struct netdev_stat_ops *ops = netdev->stat_ops;
769 int i, err;
770
771 if (!(netdev->flags & IFF_UP))
772 return 0;
773
774 i = ctx->rxq_idx;
775 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
776 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
777 i, info);
778 if (err)
779 return err;
780 ctx->rxq_idx = ++i;
781 }
782 i = ctx->txq_idx;
783 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
784 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
785 i, info);
786 if (err)
787 return err;
788 ctx->txq_idx = ++i;
789 }
790
791 ctx->rxq_idx = 0;
792 ctx->txq_idx = 0;
793 return 0;
794 }
795
796 /**
797 * netdev_stat_queue_sum() - add up queue stats from range of queues
798 * @netdev: net_device
799 * @rx_start: index of the first Rx queue to query
800 * @rx_end: index after the last Rx queue (first *not* to query)
801 * @rx_sum: output Rx stats, should be already initialized
802 * @tx_start: index of the first Tx queue to query
803 * @tx_end: index after the last Tx queue (first *not* to query)
804 * @tx_sum: output Tx stats, should be already initialized
805 *
806 * Add stats from [start, end) range of queue IDs to *x_sum structs.
807 * The sum structs must be already initialized. Usually this
808 * helper is invoked from the .get_base_stats callbacks of drivers
809 * to account for stats of disabled queues. In that case the ranges
810 * are usually [netdev->real_num_*x_queues, netdev->num_*x_queues).
811 */
netdev_stat_queue_sum(struct net_device * netdev,int rx_start,int rx_end,struct netdev_queue_stats_rx * rx_sum,int tx_start,int tx_end,struct netdev_queue_stats_tx * tx_sum)812 void netdev_stat_queue_sum(struct net_device *netdev,
813 int rx_start, int rx_end,
814 struct netdev_queue_stats_rx *rx_sum,
815 int tx_start, int tx_end,
816 struct netdev_queue_stats_tx *tx_sum)
817 {
818 const struct netdev_stat_ops *ops;
819 struct netdev_queue_stats_rx rx;
820 struct netdev_queue_stats_tx tx;
821 int i;
822
823 ops = netdev->stat_ops;
824
825 for (i = rx_start; i < rx_end; i++) {
826 memset(&rx, 0xff, sizeof(rx));
827 if (ops->get_queue_stats_rx)
828 ops->get_queue_stats_rx(netdev, i, &rx);
829 netdev_nl_stats_add(rx_sum, &rx, sizeof(rx));
830 }
831 for (i = tx_start; i < tx_end; i++) {
832 memset(&tx, 0xff, sizeof(tx));
833 if (ops->get_queue_stats_tx)
834 ops->get_queue_stats_tx(netdev, i, &tx);
835 netdev_nl_stats_add(tx_sum, &tx, sizeof(tx));
836 }
837 }
838 EXPORT_SYMBOL(netdev_stat_queue_sum);
839
840 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)841 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
842 const struct genl_info *info)
843 {
844 struct netdev_queue_stats_rx rx_sum;
845 struct netdev_queue_stats_tx tx_sum;
846 void *hdr;
847
848 /* Netdev can't guarantee any complete counters */
849 if (!netdev->stat_ops->get_base_stats)
850 return 0;
851
852 memset(&rx_sum, 0xff, sizeof(rx_sum));
853 memset(&tx_sum, 0xff, sizeof(tx_sum));
854
855 netdev->stat_ops->get_base_stats(netdev, &rx_sum, &tx_sum);
856
857 /* The op was there, but nothing reported, don't bother */
858 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
859 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
860 return 0;
861
862 hdr = genlmsg_iput(rsp, info);
863 if (!hdr)
864 return -EMSGSIZE;
865 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
866 goto nla_put_failure;
867
868 netdev_stat_queue_sum(netdev, 0, netdev->real_num_rx_queues, &rx_sum,
869 0, netdev->real_num_tx_queues, &tx_sum);
870
871 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
872 netdev_nl_stats_write_tx(rsp, &tx_sum))
873 goto nla_put_failure;
874
875 genlmsg_end(rsp, hdr);
876 return 0;
877
878 nla_put_failure:
879 genlmsg_cancel(rsp, hdr);
880 return -EMSGSIZE;
881 }
882
883 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)884 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
885 struct sk_buff *skb, const struct genl_info *info,
886 struct netdev_nl_dump_ctx *ctx)
887 {
888 if (!netdev->stat_ops)
889 return 0;
890
891 switch (scope) {
892 case 0:
893 return netdev_nl_stats_by_netdev(netdev, skb, info);
894 case NETDEV_QSTATS_SCOPE_QUEUE:
895 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
896 }
897
898 return -EINVAL; /* Should not happen, per netlink policy */
899 }
900
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)901 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
902 struct netlink_callback *cb)
903 {
904 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
905 const struct genl_info *info = genl_info_dump(cb);
906 struct net *net = sock_net(skb->sk);
907 struct net_device *netdev;
908 unsigned int ifindex;
909 unsigned int scope;
910 int err = 0;
911
912 scope = 0;
913 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
914 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
915
916 ifindex = 0;
917 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
918 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
919
920 if (ifindex) {
921 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex);
922 if (!netdev) {
923 NL_SET_BAD_ATTR(info->extack,
924 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
925 return -ENODEV;
926 }
927 if (netdev->stat_ops) {
928 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
929 info, ctx);
930 } else {
931 NL_SET_BAD_ATTR(info->extack,
932 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
933 err = -EOPNOTSUPP;
934 }
935 netdev_unlock_ops_compat(netdev);
936 return err;
937 }
938
939 for_each_netdev_lock_ops_compat_scoped(net, netdev, ctx->ifindex) {
940 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
941 info, ctx);
942 if (err < 0)
943 break;
944 }
945
946 return err;
947 }
948
netdev_nl_read_rxq_bitmap(struct genl_info * info,u32 rxq_bitmap_len,unsigned long * rxq_bitmap)949 static int netdev_nl_read_rxq_bitmap(struct genl_info *info,
950 u32 rxq_bitmap_len,
951 unsigned long *rxq_bitmap)
952 {
953 const int maxtype = ARRAY_SIZE(netdev_queue_id_nl_policy) - 1;
954 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
955 struct nlattr *attr;
956 int rem, err = 0;
957 u32 rxq_idx;
958
959 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
960 genlmsg_data(info->genlhdr),
961 genlmsg_len(info->genlhdr), rem) {
962 err = nla_parse_nested(tb, maxtype, attr,
963 netdev_queue_id_nl_policy, info->extack);
964 if (err < 0)
965 return err;
966
967 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
968 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE))
969 return -EINVAL;
970
971 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
972 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
973 return -EINVAL;
974 }
975
976 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
977 if (rxq_idx >= rxq_bitmap_len) {
978 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_ID]);
979 return -EINVAL;
980 }
981
982 bitmap_set(rxq_bitmap, rxq_idx, 1);
983 }
984
985 return 0;
986 }
987
988 static struct device *
netdev_nl_get_dma_dev(struct net_device * netdev,unsigned long * rxq_bitmap,struct netlink_ext_ack * extack)989 netdev_nl_get_dma_dev(struct net_device *netdev, unsigned long *rxq_bitmap,
990 struct netlink_ext_ack *extack)
991 {
992 struct device *dma_dev = NULL;
993 u32 rxq_idx, prev_rxq_idx;
994
995 for_each_set_bit(rxq_idx, rxq_bitmap, netdev->real_num_rx_queues) {
996 struct device *rxq_dma_dev;
997
998 rxq_dma_dev = netdev_queue_get_dma_dev(netdev, rxq_idx,
999 NETDEV_QUEUE_TYPE_RX);
1000 if (dma_dev && rxq_dma_dev != dma_dev) {
1001 NL_SET_ERR_MSG_FMT(extack, "DMA device mismatch between queue %u and %u (multi-PF device?)",
1002 rxq_idx, prev_rxq_idx);
1003 return ERR_PTR(-EOPNOTSUPP);
1004 }
1005
1006 dma_dev = rxq_dma_dev;
1007 prev_rxq_idx = rxq_idx;
1008 }
1009
1010 return dma_dev;
1011 }
1012
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)1013 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
1014 {
1015 struct net_devmem_dmabuf_binding *binding;
1016 unsigned int niov_shift = PAGE_SHIFT;
1017 u32 ifindex, dmabuf_fd, rxq_idx;
1018 struct netdev_nl_sock *priv;
1019 struct net_device *netdev;
1020 unsigned long *rxq_bitmap;
1021 struct device *dma_dev;
1022 struct sk_buff *rsp;
1023 int err = 0;
1024 void *hdr;
1025
1026 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
1027 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
1028 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
1029 return -EINVAL;
1030
1031 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
1032 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
1033
1034 if (info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]) {
1035 u32 rx_page_size = nla_get_u32(info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]);
1036
1037 if (!is_power_of_2(rx_page_size)) {
1038 NL_SET_ERR_MSG_ATTR(info->extack,
1039 info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE],
1040 "rx-page-size must be a power of 2");
1041 return -EINVAL;
1042 }
1043 niov_shift = ilog2(rx_page_size);
1044 }
1045
1046 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
1047 if (IS_ERR(priv))
1048 return PTR_ERR(priv);
1049
1050 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1051 if (!rsp)
1052 return -ENOMEM;
1053
1054 hdr = genlmsg_iput(rsp, info);
1055 if (!hdr) {
1056 err = -EMSGSIZE;
1057 goto err_genlmsg_free;
1058 }
1059
1060 mutex_lock(&priv->lock);
1061
1062 err = 0;
1063 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
1064 if (!netdev) {
1065 err = -ENODEV;
1066 goto err_unlock_sock;
1067 }
1068 if (!netif_device_present(netdev))
1069 err = -ENODEV;
1070 else if (!netdev_need_ops_lock(netdev))
1071 err = -EOPNOTSUPP;
1072 if (err) {
1073 NL_SET_BAD_ATTR(info->extack,
1074 info->attrs[NETDEV_A_DEV_IFINDEX]);
1075 goto err_unlock;
1076 }
1077
1078 rxq_bitmap = bitmap_zalloc(netdev->real_num_rx_queues, GFP_KERNEL);
1079 if (!rxq_bitmap) {
1080 err = -ENOMEM;
1081 goto err_unlock;
1082 }
1083
1084 err = netdev_nl_read_rxq_bitmap(info, netdev->real_num_rx_queues,
1085 rxq_bitmap);
1086 if (err)
1087 goto err_rxq_bitmap;
1088
1089 dma_dev = netdev_nl_get_dma_dev(netdev, rxq_bitmap, info->extack);
1090 if (IS_ERR(dma_dev)) {
1091 err = PTR_ERR(dma_dev);
1092 goto err_rxq_bitmap;
1093 }
1094
1095 binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE,
1096 dmabuf_fd, niov_shift, priv,
1097 info->extack);
1098 if (IS_ERR(binding)) {
1099 err = PTR_ERR(binding);
1100 goto err_rxq_bitmap;
1101 }
1102
1103 for_each_set_bit(rxq_idx, rxq_bitmap, netdev->real_num_rx_queues) {
1104 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
1105 info->extack);
1106 if (err)
1107 goto err_unbind;
1108 }
1109
1110 /* rsp was allocated large enough */
1111 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id));
1112
1113 genlmsg_end(rsp, hdr);
1114
1115 err = genlmsg_reply(rsp, info);
1116
1117 bitmap_free(rxq_bitmap);
1118
1119 netdev_unlock(netdev);
1120
1121 mutex_unlock(&priv->lock);
1122
1123 return err < 0 ? err : 0;
1124
1125 err_unbind:
1126 net_devmem_unbind_dmabuf(binding);
1127 err_rxq_bitmap:
1128 bitmap_free(rxq_bitmap);
1129 err_unlock:
1130 netdev_unlock(netdev);
1131 err_unlock_sock:
1132 mutex_unlock(&priv->lock);
1133 err_genlmsg_free:
1134 nlmsg_free(rsp);
1135 return err;
1136 }
1137
1138 /* Find the DMA-capable device for a netmem TX binding.
1139 *
1140 * For NETMEM_TX_DMA devices, return the device itself.
1141 * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying
1142 * physical device and return it.
1143 */
1144 static struct net_device *
netdev_find_netmem_tx_dev(struct net_device * dev)1145 netdev_find_netmem_tx_dev(struct net_device *dev)
1146 {
1147 struct netdev_rx_queue *lease_rxq;
1148 struct net_device *phys_dev;
1149 int i;
1150
1151 if (dev->netmem_tx == NETMEM_TX_DMA)
1152 return dev;
1153
1154 if (dev->netmem_tx != NETMEM_TX_NO_DMA)
1155 return NULL;
1156
1157 for (i = 0; i < dev->real_num_rx_queues; i++) {
1158 lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease);
1159 if (!lease_rxq)
1160 continue;
1161
1162 phys_dev = lease_rxq->dev;
1163 if (netif_device_present(phys_dev) &&
1164 phys_dev->netmem_tx == NETMEM_TX_DMA)
1165 return phys_dev;
1166 }
1167
1168 return NULL;
1169 }
1170
netdev_nl_bind_tx_doit(struct sk_buff * skb,struct genl_info * info)1171 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info)
1172 {
1173 struct net_devmem_dmabuf_binding *binding;
1174 struct net_device *bind_dev;
1175 struct netdev_nl_sock *priv;
1176 struct net_device *netdev;
1177 struct device *dma_dev;
1178 u32 ifindex, dmabuf_fd;
1179 struct sk_buff *rsp;
1180 int err = 0;
1181 void *hdr;
1182
1183 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
1184 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD))
1185 return -EINVAL;
1186
1187 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
1188 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
1189
1190 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
1191 if (IS_ERR(priv))
1192 return PTR_ERR(priv);
1193
1194 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1195 if (!rsp)
1196 return -ENOMEM;
1197
1198 hdr = genlmsg_iput(rsp, info);
1199 if (!hdr) {
1200 err = -EMSGSIZE;
1201 goto err_genlmsg_free;
1202 }
1203
1204 mutex_lock(&priv->lock);
1205
1206 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
1207 if (!netdev) {
1208 err = -ENODEV;
1209 goto err_unlock_sock;
1210 }
1211
1212 if (!netif_device_present(netdev)) {
1213 err = -ENODEV;
1214 goto err_unlock_netdev;
1215 }
1216
1217 if (netdev->netmem_tx == NETMEM_TX_NONE) {
1218 err = -EOPNOTSUPP;
1219 NL_SET_ERR_MSG(info->extack,
1220 "Driver does not support netmem TX");
1221 goto err_unlock_netdev;
1222 }
1223
1224 bind_dev = netdev_find_netmem_tx_dev(netdev);
1225 if (!bind_dev) {
1226 err = -EOPNOTSUPP;
1227 NL_SET_ERR_MSG(info->extack,
1228 "No DMA-capable device found for netmem TX");
1229 goto err_unlock_netdev;
1230 }
1231
1232 if (bind_dev != netdev)
1233 netdev_lock(bind_dev);
1234
1235 dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX);
1236
1237 binding = net_devmem_bind_dmabuf(bind_dev,
1238 bind_dev != netdev ? netdev : NULL,
1239 dma_dev, DMA_TO_DEVICE, dmabuf_fd,
1240 PAGE_SHIFT, priv, info->extack);
1241 if (IS_ERR(binding)) {
1242 err = PTR_ERR(binding);
1243 goto err_unlock_bind_dev;
1244 }
1245
1246 /* rsp was allocated large enough */
1247 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id));
1248
1249 genlmsg_end(rsp, hdr);
1250
1251 if (bind_dev != netdev)
1252 netdev_unlock(bind_dev);
1253 netdev_unlock(netdev);
1254 mutex_unlock(&priv->lock);
1255
1256 return genlmsg_reply(rsp, info);
1257
1258 err_unlock_bind_dev:
1259 if (bind_dev != netdev)
1260 netdev_unlock(bind_dev);
1261 err_unlock_netdev:
1262 netdev_unlock(netdev);
1263 err_unlock_sock:
1264 mutex_unlock(&priv->lock);
1265 err_genlmsg_free:
1266 nlmsg_free(rsp);
1267 return err;
1268 }
1269
netdev_nl_queue_create_doit(struct sk_buff * skb,struct genl_info * info)1270 int netdev_nl_queue_create_doit(struct sk_buff *skb, struct genl_info *info)
1271 {
1272 const int qmaxtype = ARRAY_SIZE(netdev_queue_id_nl_policy) - 1;
1273 const int lmaxtype = ARRAY_SIZE(netdev_lease_nl_policy) - 1;
1274 int err, ifindex, ifindex_lease, queue_id, queue_id_lease;
1275 struct nlattr *qtb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
1276 struct nlattr *ltb[ARRAY_SIZE(netdev_lease_nl_policy)];
1277 struct netdev_rx_queue *rxq, *rxq_lease;
1278 struct net_device *dev, *dev_lease;
1279 netdevice_tracker dev_tracker;
1280 s32 netns_lease = -1;
1281 struct nlattr *nest;
1282 struct sk_buff *rsp;
1283 struct net *net;
1284 void *hdr;
1285
1286 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX) ||
1287 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
1288 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_LEASE))
1289 return -EINVAL;
1290 if (nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]) !=
1291 NETDEV_QUEUE_TYPE_RX) {
1292 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_QUEUE_TYPE]);
1293 return -EINVAL;
1294 }
1295
1296 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
1297
1298 nest = info->attrs[NETDEV_A_QUEUE_LEASE];
1299 err = nla_parse_nested(ltb, lmaxtype, nest,
1300 netdev_lease_nl_policy, info->extack);
1301 if (err < 0)
1302 return err;
1303 if (NL_REQ_ATTR_CHECK(info->extack, nest, ltb, NETDEV_A_LEASE_IFINDEX) ||
1304 NL_REQ_ATTR_CHECK(info->extack, nest, ltb, NETDEV_A_LEASE_QUEUE))
1305 return -EINVAL;
1306 if (ltb[NETDEV_A_LEASE_NETNS_ID]) {
1307 if (!capable(CAP_NET_ADMIN))
1308 return -EPERM;
1309 netns_lease = nla_get_s32(ltb[NETDEV_A_LEASE_NETNS_ID]);
1310 }
1311
1312 ifindex_lease = nla_get_u32(ltb[NETDEV_A_LEASE_IFINDEX]);
1313
1314 nest = ltb[NETDEV_A_LEASE_QUEUE];
1315 err = nla_parse_nested(qtb, qmaxtype, nest,
1316 netdev_queue_id_nl_policy, info->extack);
1317 if (err < 0)
1318 return err;
1319 if (NL_REQ_ATTR_CHECK(info->extack, nest, qtb, NETDEV_A_QUEUE_ID) ||
1320 NL_REQ_ATTR_CHECK(info->extack, nest, qtb, NETDEV_A_QUEUE_TYPE))
1321 return -EINVAL;
1322 if (nla_get_u32(qtb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
1323 NL_SET_BAD_ATTR(info->extack, qtb[NETDEV_A_QUEUE_TYPE]);
1324 return -EINVAL;
1325 }
1326
1327 queue_id_lease = nla_get_u32(qtb[NETDEV_A_QUEUE_ID]);
1328
1329 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1330 if (!rsp)
1331 return -ENOMEM;
1332
1333 hdr = genlmsg_iput(rsp, info);
1334 if (!hdr) {
1335 err = -EMSGSIZE;
1336 goto err_genlmsg_free;
1337 }
1338
1339 /* Locking order is always from the virtual to the physical device
1340 * since this is also the same order when applications open the
1341 * memory provider later on.
1342 */
1343 dev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
1344 if (!dev) {
1345 err = -ENODEV;
1346 goto err_genlmsg_free;
1347 }
1348 if (!netdev_can_create_queue(dev, info->extack)) {
1349 err = -EINVAL;
1350 goto err_unlock_dev;
1351 }
1352
1353 net = genl_info_net(info);
1354 if (netns_lease >= 0) {
1355 net = get_net_ns_by_id(net, netns_lease);
1356 if (!net) {
1357 err = -ENONET;
1358 goto err_unlock_dev;
1359 }
1360 }
1361
1362 dev_lease = netdev_get_by_index(net, ifindex_lease, &dev_tracker,
1363 GFP_KERNEL);
1364 if (!dev_lease) {
1365 err = -ENODEV;
1366 goto err_put_netns;
1367 }
1368 if (!netdev_can_lease_queue(dev_lease, info->extack)) {
1369 netdev_put(dev_lease, &dev_tracker);
1370 err = -EINVAL;
1371 goto err_put_netns;
1372 }
1373
1374 dev_lease = netdev_put_lock(dev_lease, net, &dev_tracker);
1375 if (!dev_lease) {
1376 err = -ENODEV;
1377 goto err_put_netns;
1378 }
1379 if (queue_id_lease >= dev_lease->real_num_rx_queues) {
1380 err = -ERANGE;
1381 NL_SET_BAD_ATTR(info->extack, qtb[NETDEV_A_QUEUE_ID]);
1382 goto err_unlock_dev_lease;
1383 }
1384 if (netdev_queue_busy(dev_lease, queue_id_lease, NETDEV_QUEUE_TYPE_RX,
1385 info->extack)) {
1386 err = -EBUSY;
1387 goto err_unlock_dev_lease;
1388 }
1389
1390 rxq_lease = __netif_get_rx_queue(dev_lease, queue_id_lease);
1391 rxq = __netif_get_rx_queue(dev, dev->real_num_rx_queues - 1);
1392
1393 /* Leasing queues from different physical devices is currently
1394 * not supported. Capabilities such as XDP features and DMA
1395 * device may differ between physical devices, and computing
1396 * a correct intersection for the virtual device is not yet
1397 * implemented.
1398 */
1399 if (rxq->lease && rxq->lease->dev != dev_lease) {
1400 err = -EOPNOTSUPP;
1401 NL_SET_ERR_MSG(info->extack,
1402 "Leasing queues from different devices not supported");
1403 goto err_unlock_dev_lease;
1404 }
1405
1406 queue_id = dev->queue_mgmt_ops->ndo_queue_create(dev, info->extack);
1407 if (queue_id < 0) {
1408 err = queue_id;
1409 goto err_unlock_dev_lease;
1410 }
1411 rxq = __netif_get_rx_queue(dev, queue_id);
1412
1413 netdev_rx_queue_lease(rxq, rxq_lease);
1414
1415 /* rsp was allocated large enough */
1416 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_QUEUE_ID, queue_id));
1417
1418 genlmsg_end(rsp, hdr);
1419
1420 netdev_unlock(dev_lease);
1421 netdev_unlock(dev);
1422 if (netns_lease >= 0)
1423 put_net(net);
1424
1425 return genlmsg_reply(rsp, info);
1426
1427 err_unlock_dev_lease:
1428 netdev_unlock(dev_lease);
1429 err_put_netns:
1430 if (netns_lease >= 0)
1431 put_net(net);
1432 err_unlock_dev:
1433 netdev_unlock(dev);
1434 err_genlmsg_free:
1435 nlmsg_free(rsp);
1436 return err;
1437 }
1438
netdev_nl_sock_priv_init(struct netdev_nl_sock * priv)1439 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv)
1440 {
1441 INIT_LIST_HEAD(&priv->bindings);
1442 mutex_init(&priv->lock);
1443 }
1444
netdev_nl_sock_priv_destroy(struct netdev_nl_sock * priv)1445 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv)
1446 {
1447 struct net_devmem_dmabuf_binding *binding;
1448 struct net_devmem_dmabuf_binding *temp;
1449 netdevice_tracker dev_tracker;
1450 struct net_device *dev;
1451
1452 mutex_lock(&priv->lock);
1453 list_for_each_entry_safe(binding, temp, &priv->bindings, list) {
1454 mutex_lock(&binding->lock);
1455 dev = binding->dev;
1456 if (!dev) {
1457 mutex_unlock(&binding->lock);
1458 net_devmem_unbind_dmabuf(binding);
1459 continue;
1460 }
1461 netdev_hold(dev, &dev_tracker, GFP_KERNEL);
1462 mutex_unlock(&binding->lock);
1463
1464 netdev_lock(dev);
1465 net_devmem_unbind_dmabuf(binding);
1466 netdev_unlock(dev);
1467 netdev_put(dev, &dev_tracker);
1468 }
1469 mutex_unlock(&priv->lock);
1470 }
1471
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)1472 static int netdev_genl_netdevice_event(struct notifier_block *nb,
1473 unsigned long event, void *ptr)
1474 {
1475 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
1476
1477 switch (event) {
1478 case NETDEV_REGISTER:
1479 netdev_lock_ops_to_full(netdev);
1480 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
1481 netdev_unlock_full_to_ops(netdev);
1482 break;
1483 case NETDEV_UNREGISTER:
1484 netdev_lock(netdev);
1485 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
1486 netdev_unlock(netdev);
1487 break;
1488 case NETDEV_XDP_FEAT_CHANGE:
1489 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
1490 break;
1491 }
1492
1493 return NOTIFY_OK;
1494 }
1495
1496 static struct notifier_block netdev_genl_nb = {
1497 .notifier_call = netdev_genl_netdevice_event,
1498 };
1499
netdev_genl_init(void)1500 static int __init netdev_genl_init(void)
1501 {
1502 int err;
1503
1504 err = register_netdevice_notifier(&netdev_genl_nb);
1505 if (err)
1506 return err;
1507
1508 err = genl_register_family(&netdev_nl_family);
1509 if (err)
1510 goto err_unreg_ntf;
1511
1512 return 0;
1513
1514 err_unreg_ntf:
1515 unregister_netdevice_notifier(&netdev_genl_nb);
1516 return err;
1517 }
1518
1519 subsys_initcall(netdev_genl_init);
1520