1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/rtnetlink.h>
6 #include <net/busy_poll.h>
7 #include <net/net_namespace.h>
8 #include <net/netdev_queues.h>
9 #include <net/netdev_rx_queue.h>
10 #include <net/sock.h>
11 #include <net/xdp.h>
12 #include <net/xdp_sock.h>
13 #include <net/page_pool/memory_provider.h>
14
15 #include "dev.h"
16 #include "devmem.h"
17 #include "netdev-genl-gen.h"
18
19 struct netdev_nl_dump_ctx {
20 unsigned long ifindex;
21 unsigned int rxq_idx;
22 unsigned int txq_idx;
23 unsigned int napi_id;
24 };
25
netdev_dump_ctx(struct netlink_callback * cb)26 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
27 {
28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
29
30 return (struct netdev_nl_dump_ctx *)cb->ctx;
31 }
32
33 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)34 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
35 const struct genl_info *info)
36 {
37 u64 xsk_features = 0;
38 u64 xdp_rx_meta = 0;
39 void *hdr;
40
41 netdev_assert_locked(netdev); /* note: rtnl_lock may not be held! */
42
43 hdr = genlmsg_iput(rsp, info);
44 if (!hdr)
45 return -EMSGSIZE;
46
47 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
48 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
49 xdp_rx_meta |= flag;
50 XDP_METADATA_KFUNC_xxx
51 #undef XDP_METADATA_KFUNC
52
53 if (netdev->xsk_tx_metadata_ops) {
54 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
55 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
56 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
57 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
58 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time)
59 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO;
60 }
61
62 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
64 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
65 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
66 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
67 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
68 xsk_features, NETDEV_A_DEV_PAD))
69 goto err_cancel_msg;
70
71 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
72 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
73 netdev->xdp_zc_max_segs))
74 goto err_cancel_msg;
75 }
76
77 genlmsg_end(rsp, hdr);
78
79 return 0;
80
81 err_cancel_msg:
82 genlmsg_cancel(rsp, hdr);
83 return -EMSGSIZE;
84 }
85
86 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)87 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
88 {
89 struct genl_info info;
90 struct sk_buff *ntf;
91
92 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
93 NETDEV_NLGRP_MGMT))
94 return;
95
96 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
97
98 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
99 if (!ntf)
100 return;
101
102 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
103 nlmsg_free(ntf);
104 return;
105 }
106
107 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
108 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
109 }
110
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)111 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
112 {
113 struct net_device *netdev;
114 struct sk_buff *rsp;
115 u32 ifindex;
116 int err;
117
118 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
119 return -EINVAL;
120
121 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
122
123 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
124 if (!rsp)
125 return -ENOMEM;
126
127 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
128 if (!netdev) {
129 err = -ENODEV;
130 goto err_free_msg;
131 }
132
133 err = netdev_nl_dev_fill(netdev, rsp, info);
134 netdev_unlock(netdev);
135
136 if (err)
137 goto err_free_msg;
138
139 return genlmsg_reply(rsp, info);
140
141 err_free_msg:
142 nlmsg_free(rsp);
143 return err;
144 }
145
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)146 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
147 {
148 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
149 struct net *net = sock_net(skb->sk);
150 int err;
151
152 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
153 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
154 if (err < 0)
155 return err;
156 }
157
158 return 0;
159 }
160
161 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)162 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
163 const struct genl_info *info)
164 {
165 unsigned long irq_suspend_timeout;
166 unsigned long gro_flush_timeout;
167 u32 napi_defer_hard_irqs;
168 void *hdr;
169 pid_t pid;
170
171 if (!napi->dev->up)
172 return 0;
173
174 hdr = genlmsg_iput(rsp, info);
175 if (!hdr)
176 return -EMSGSIZE;
177
178 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
179 goto nla_put_failure;
180
181 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
182 goto nla_put_failure;
183
184 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
185 goto nla_put_failure;
186
187 if (napi->thread) {
188 pid = task_pid_nr(napi->thread);
189 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
190 goto nla_put_failure;
191 }
192
193 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
194 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
195 napi_defer_hard_irqs))
196 goto nla_put_failure;
197
198 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
199 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
200 irq_suspend_timeout))
201 goto nla_put_failure;
202
203 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
204 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
205 gro_flush_timeout))
206 goto nla_put_failure;
207
208 genlmsg_end(rsp, hdr);
209
210 return 0;
211
212 nla_put_failure:
213 genlmsg_cancel(rsp, hdr);
214 return -EMSGSIZE;
215 }
216
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)217 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
218 {
219 struct napi_struct *napi;
220 struct sk_buff *rsp;
221 u32 napi_id;
222 int err;
223
224 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
225 return -EINVAL;
226
227 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
228
229 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
230 if (!rsp)
231 return -ENOMEM;
232
233 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
234 if (napi) {
235 err = netdev_nl_napi_fill_one(rsp, napi, info);
236 netdev_unlock(napi->dev);
237 } else {
238 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
239 err = -ENOENT;
240 }
241
242 if (err) {
243 goto err_free_msg;
244 } else if (!rsp->len) {
245 err = -ENOENT;
246 goto err_free_msg;
247 }
248
249 return genlmsg_reply(rsp, info);
250
251 err_free_msg:
252 nlmsg_free(rsp);
253 return err;
254 }
255
256 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)257 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
258 const struct genl_info *info,
259 struct netdev_nl_dump_ctx *ctx)
260 {
261 struct napi_struct *napi;
262 unsigned int prev_id;
263 int err = 0;
264
265 if (!netdev->up)
266 return err;
267
268 prev_id = UINT_MAX;
269 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
270 if (!napi_id_valid(napi->napi_id))
271 continue;
272
273 /* Dump continuation below depends on the list being sorted */
274 WARN_ON_ONCE(napi->napi_id >= prev_id);
275 prev_id = napi->napi_id;
276
277 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
278 continue;
279
280 err = netdev_nl_napi_fill_one(rsp, napi, info);
281 if (err)
282 return err;
283 ctx->napi_id = napi->napi_id;
284 }
285 return err;
286 }
287
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)288 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
289 {
290 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
291 const struct genl_info *info = genl_info_dump(cb);
292 struct net *net = sock_net(skb->sk);
293 struct net_device *netdev;
294 u32 ifindex = 0;
295 int err = 0;
296
297 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
298 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
299
300 if (ifindex) {
301 netdev = netdev_get_by_index_lock(net, ifindex);
302 if (netdev) {
303 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
304 netdev_unlock(netdev);
305 } else {
306 err = -ENODEV;
307 }
308 } else {
309 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
310 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
311 if (err < 0)
312 break;
313 ctx->napi_id = 0;
314 }
315 }
316
317 return err;
318 }
319
320 static int
netdev_nl_napi_set_config(struct napi_struct * napi,struct genl_info * info)321 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
322 {
323 u64 irq_suspend_timeout = 0;
324 u64 gro_flush_timeout = 0;
325 u32 defer = 0;
326
327 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
328 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
329 napi_set_defer_hard_irqs(napi, defer);
330 }
331
332 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
333 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
334 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
335 }
336
337 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
338 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
339 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
340 }
341
342 return 0;
343 }
344
netdev_nl_napi_set_doit(struct sk_buff * skb,struct genl_info * info)345 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
346 {
347 struct napi_struct *napi;
348 unsigned int napi_id;
349 int err;
350
351 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
352 return -EINVAL;
353
354 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
355
356 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
357 if (napi) {
358 err = netdev_nl_napi_set_config(napi, info);
359 netdev_unlock(napi->dev);
360 } else {
361 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
362 err = -ENOENT;
363 }
364
365 return err;
366 }
367
nla_put_napi_id(struct sk_buff * skb,const struct napi_struct * napi)368 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi)
369 {
370 if (napi && napi_id_valid(napi->napi_id))
371 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id);
372 return 0;
373 }
374
375 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)376 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
377 u32 q_idx, u32 q_type, const struct genl_info *info)
378 {
379 struct pp_memory_provider_params *params;
380 struct netdev_rx_queue *rxq;
381 struct netdev_queue *txq;
382 void *hdr;
383
384 hdr = genlmsg_iput(rsp, info);
385 if (!hdr)
386 return -EMSGSIZE;
387
388 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
389 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
390 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
391 goto nla_put_failure;
392
393 switch (q_type) {
394 case NETDEV_QUEUE_TYPE_RX:
395 rxq = __netif_get_rx_queue(netdev, q_idx);
396 if (nla_put_napi_id(rsp, rxq->napi))
397 goto nla_put_failure;
398
399 params = &rxq->mp_params;
400 if (params->mp_ops &&
401 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq))
402 goto nla_put_failure;
403 #ifdef CONFIG_XDP_SOCKETS
404 if (rxq->pool)
405 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
406 goto nla_put_failure;
407 #endif
408
409 break;
410 case NETDEV_QUEUE_TYPE_TX:
411 txq = netdev_get_tx_queue(netdev, q_idx);
412 if (nla_put_napi_id(rsp, txq->napi))
413 goto nla_put_failure;
414 #ifdef CONFIG_XDP_SOCKETS
415 if (txq->pool)
416 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
417 goto nla_put_failure;
418 #endif
419 break;
420 }
421
422 genlmsg_end(rsp, hdr);
423
424 return 0;
425
426 nla_put_failure:
427 genlmsg_cancel(rsp, hdr);
428 return -EMSGSIZE;
429 }
430
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)431 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
432 u32 q_type)
433 {
434 switch (q_type) {
435 case NETDEV_QUEUE_TYPE_RX:
436 if (q_id >= netdev->real_num_rx_queues)
437 return -EINVAL;
438 return 0;
439 case NETDEV_QUEUE_TYPE_TX:
440 if (q_id >= netdev->real_num_tx_queues)
441 return -EINVAL;
442 }
443 return 0;
444 }
445
446 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)447 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
448 u32 q_type, const struct genl_info *info)
449 {
450 int err;
451
452 if (!netdev->up)
453 return -ENOENT;
454
455 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
456 if (err)
457 return err;
458
459 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
460 }
461
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)462 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
463 {
464 u32 q_id, q_type, ifindex;
465 struct net_device *netdev;
466 struct sk_buff *rsp;
467 int err;
468
469 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
470 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
471 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
472 return -EINVAL;
473
474 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
475 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
476 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
477
478 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
479 if (!rsp)
480 return -ENOMEM;
481
482 netdev = netdev_get_by_index_lock_ops_compat(genl_info_net(info),
483 ifindex);
484 if (netdev) {
485 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
486 netdev_unlock_ops_compat(netdev);
487 } else {
488 err = -ENODEV;
489 }
490
491 if (err)
492 goto err_free_msg;
493
494 return genlmsg_reply(rsp, info);
495
496 err_free_msg:
497 nlmsg_free(rsp);
498 return err;
499 }
500
501 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)502 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
503 const struct genl_info *info,
504 struct netdev_nl_dump_ctx *ctx)
505 {
506 int err = 0;
507
508 if (!netdev->up)
509 return err;
510
511 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
512 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
513 NETDEV_QUEUE_TYPE_RX, info);
514 if (err)
515 return err;
516 }
517 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
518 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
519 NETDEV_QUEUE_TYPE_TX, info);
520 if (err)
521 return err;
522 }
523
524 return err;
525 }
526
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)527 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
528 {
529 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
530 const struct genl_info *info = genl_info_dump(cb);
531 struct net *net = sock_net(skb->sk);
532 struct net_device *netdev;
533 u32 ifindex = 0;
534 int err = 0;
535
536 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
537 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
538
539 if (ifindex) {
540 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex);
541 if (netdev) {
542 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
543 netdev_unlock_ops_compat(netdev);
544 } else {
545 err = -ENODEV;
546 }
547 } else {
548 for_each_netdev_lock_ops_compat_scoped(net, netdev,
549 ctx->ifindex) {
550 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
551 if (err < 0)
552 break;
553 ctx->rxq_idx = 0;
554 ctx->txq_idx = 0;
555 }
556 }
557
558 return err;
559 }
560
561 #define NETDEV_STAT_NOT_SET (~0ULL)
562
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)563 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
564 {
565 const u64 *add = _add;
566 u64 *sum = _sum;
567
568 while (size) {
569 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
570 *sum += *add;
571 sum++;
572 add++;
573 size -= 8;
574 }
575 }
576
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)577 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
578 {
579 if (value == NETDEV_STAT_NOT_SET)
580 return 0;
581 return nla_put_uint(rsp, attr_id, value);
582 }
583
584 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)585 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
586 {
587 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
588 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
589 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
590 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
591 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
592 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) ||
593 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
594 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
595 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
596 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
597 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
598 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
599 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
600 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
601 return -EMSGSIZE;
602 return 0;
603 }
604
605 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)606 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
607 {
608 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
609 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
610 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
611 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
612 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
613 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
614 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
615 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
616 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
617 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
618 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
619 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
620 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
621 return -EMSGSIZE;
622 return 0;
623 }
624
625 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)626 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
627 u32 q_type, int i, const struct genl_info *info)
628 {
629 const struct netdev_stat_ops *ops = netdev->stat_ops;
630 struct netdev_queue_stats_rx rx;
631 struct netdev_queue_stats_tx tx;
632 void *hdr;
633
634 hdr = genlmsg_iput(rsp, info);
635 if (!hdr)
636 return -EMSGSIZE;
637 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
638 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
639 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
640 goto nla_put_failure;
641
642 switch (q_type) {
643 case NETDEV_QUEUE_TYPE_RX:
644 memset(&rx, 0xff, sizeof(rx));
645 ops->get_queue_stats_rx(netdev, i, &rx);
646 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
647 goto nla_cancel;
648 if (netdev_nl_stats_write_rx(rsp, &rx))
649 goto nla_put_failure;
650 break;
651 case NETDEV_QUEUE_TYPE_TX:
652 memset(&tx, 0xff, sizeof(tx));
653 ops->get_queue_stats_tx(netdev, i, &tx);
654 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
655 goto nla_cancel;
656 if (netdev_nl_stats_write_tx(rsp, &tx))
657 goto nla_put_failure;
658 break;
659 }
660
661 genlmsg_end(rsp, hdr);
662 return 0;
663
664 nla_cancel:
665 genlmsg_cancel(rsp, hdr);
666 return 0;
667 nla_put_failure:
668 genlmsg_cancel(rsp, hdr);
669 return -EMSGSIZE;
670 }
671
672 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)673 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
674 const struct genl_info *info,
675 struct netdev_nl_dump_ctx *ctx)
676 {
677 const struct netdev_stat_ops *ops = netdev->stat_ops;
678 int i, err;
679
680 if (!(netdev->flags & IFF_UP))
681 return 0;
682
683 i = ctx->rxq_idx;
684 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
685 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
686 i, info);
687 if (err)
688 return err;
689 ctx->rxq_idx = ++i;
690 }
691 i = ctx->txq_idx;
692 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
693 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
694 i, info);
695 if (err)
696 return err;
697 ctx->txq_idx = ++i;
698 }
699
700 ctx->rxq_idx = 0;
701 ctx->txq_idx = 0;
702 return 0;
703 }
704
705 /**
706 * netdev_stat_queue_sum() - add up queue stats from range of queues
707 * @netdev: net_device
708 * @rx_start: index of the first Rx queue to query
709 * @rx_end: index after the last Rx queue (first *not* to query)
710 * @rx_sum: output Rx stats, should be already initialized
711 * @tx_start: index of the first Tx queue to query
712 * @tx_end: index after the last Tx queue (first *not* to query)
713 * @tx_sum: output Tx stats, should be already initialized
714 *
715 * Add stats from [start, end) range of queue IDs to *x_sum structs.
716 * The sum structs must be already initialized. Usually this
717 * helper is invoked from the .get_base_stats callbacks of drivers
718 * to account for stats of disabled queues. In that case the ranges
719 * are usually [netdev->real_num_*x_queues, netdev->num_*x_queues).
720 */
netdev_stat_queue_sum(struct net_device * netdev,int rx_start,int rx_end,struct netdev_queue_stats_rx * rx_sum,int tx_start,int tx_end,struct netdev_queue_stats_tx * tx_sum)721 void netdev_stat_queue_sum(struct net_device *netdev,
722 int rx_start, int rx_end,
723 struct netdev_queue_stats_rx *rx_sum,
724 int tx_start, int tx_end,
725 struct netdev_queue_stats_tx *tx_sum)
726 {
727 const struct netdev_stat_ops *ops;
728 struct netdev_queue_stats_rx rx;
729 struct netdev_queue_stats_tx tx;
730 int i;
731
732 ops = netdev->stat_ops;
733
734 for (i = rx_start; i < rx_end; i++) {
735 memset(&rx, 0xff, sizeof(rx));
736 if (ops->get_queue_stats_rx)
737 ops->get_queue_stats_rx(netdev, i, &rx);
738 netdev_nl_stats_add(rx_sum, &rx, sizeof(rx));
739 }
740 for (i = tx_start; i < tx_end; i++) {
741 memset(&tx, 0xff, sizeof(tx));
742 if (ops->get_queue_stats_tx)
743 ops->get_queue_stats_tx(netdev, i, &tx);
744 netdev_nl_stats_add(tx_sum, &tx, sizeof(tx));
745 }
746 }
747 EXPORT_SYMBOL(netdev_stat_queue_sum);
748
749 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)750 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
751 const struct genl_info *info)
752 {
753 struct netdev_queue_stats_rx rx_sum;
754 struct netdev_queue_stats_tx tx_sum;
755 void *hdr;
756
757 /* Netdev can't guarantee any complete counters */
758 if (!netdev->stat_ops->get_base_stats)
759 return 0;
760
761 memset(&rx_sum, 0xff, sizeof(rx_sum));
762 memset(&tx_sum, 0xff, sizeof(tx_sum));
763
764 netdev->stat_ops->get_base_stats(netdev, &rx_sum, &tx_sum);
765
766 /* The op was there, but nothing reported, don't bother */
767 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
768 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
769 return 0;
770
771 hdr = genlmsg_iput(rsp, info);
772 if (!hdr)
773 return -EMSGSIZE;
774 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
775 goto nla_put_failure;
776
777 netdev_stat_queue_sum(netdev, 0, netdev->real_num_rx_queues, &rx_sum,
778 0, netdev->real_num_tx_queues, &tx_sum);
779
780 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
781 netdev_nl_stats_write_tx(rsp, &tx_sum))
782 goto nla_put_failure;
783
784 genlmsg_end(rsp, hdr);
785 return 0;
786
787 nla_put_failure:
788 genlmsg_cancel(rsp, hdr);
789 return -EMSGSIZE;
790 }
791
792 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)793 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
794 struct sk_buff *skb, const struct genl_info *info,
795 struct netdev_nl_dump_ctx *ctx)
796 {
797 if (!netdev->stat_ops)
798 return 0;
799
800 switch (scope) {
801 case 0:
802 return netdev_nl_stats_by_netdev(netdev, skb, info);
803 case NETDEV_QSTATS_SCOPE_QUEUE:
804 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
805 }
806
807 return -EINVAL; /* Should not happen, per netlink policy */
808 }
809
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)810 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
811 struct netlink_callback *cb)
812 {
813 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
814 const struct genl_info *info = genl_info_dump(cb);
815 struct net *net = sock_net(skb->sk);
816 struct net_device *netdev;
817 unsigned int ifindex;
818 unsigned int scope;
819 int err = 0;
820
821 scope = 0;
822 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
823 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
824
825 ifindex = 0;
826 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
827 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
828
829 if (ifindex) {
830 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex);
831 if (!netdev) {
832 NL_SET_BAD_ATTR(info->extack,
833 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
834 return -ENODEV;
835 }
836 if (netdev->stat_ops) {
837 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
838 info, ctx);
839 } else {
840 NL_SET_BAD_ATTR(info->extack,
841 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
842 err = -EOPNOTSUPP;
843 }
844 netdev_unlock_ops_compat(netdev);
845 return err;
846 }
847
848 for_each_netdev_lock_ops_compat_scoped(net, netdev, ctx->ifindex) {
849 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
850 info, ctx);
851 if (err < 0)
852 break;
853 }
854
855 return err;
856 }
857
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)858 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
859 {
860 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
861 struct net_devmem_dmabuf_binding *binding;
862 u32 ifindex, dmabuf_fd, rxq_idx;
863 struct netdev_nl_sock *priv;
864 struct net_device *netdev;
865 struct sk_buff *rsp;
866 struct nlattr *attr;
867 int rem, err = 0;
868 void *hdr;
869
870 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
871 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
872 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
873 return -EINVAL;
874
875 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
876 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
877
878 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
879 if (IS_ERR(priv))
880 return PTR_ERR(priv);
881
882 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
883 if (!rsp)
884 return -ENOMEM;
885
886 hdr = genlmsg_iput(rsp, info);
887 if (!hdr) {
888 err = -EMSGSIZE;
889 goto err_genlmsg_free;
890 }
891
892 mutex_lock(&priv->lock);
893
894 err = 0;
895 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
896 if (!netdev) {
897 err = -ENODEV;
898 goto err_unlock_sock;
899 }
900 if (!netif_device_present(netdev))
901 err = -ENODEV;
902 else if (!netdev_need_ops_lock(netdev))
903 err = -EOPNOTSUPP;
904 if (err) {
905 NL_SET_BAD_ATTR(info->extack,
906 info->attrs[NETDEV_A_DEV_IFINDEX]);
907 goto err_unlock;
908 }
909
910 binding = net_devmem_bind_dmabuf(netdev, DMA_FROM_DEVICE, dmabuf_fd,
911 priv, info->extack);
912 if (IS_ERR(binding)) {
913 err = PTR_ERR(binding);
914 goto err_unlock;
915 }
916
917 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
918 genlmsg_data(info->genlhdr),
919 genlmsg_len(info->genlhdr), rem) {
920 err = nla_parse_nested(
921 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
922 netdev_queue_id_nl_policy, info->extack);
923 if (err < 0)
924 goto err_unbind;
925
926 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
927 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
928 err = -EINVAL;
929 goto err_unbind;
930 }
931
932 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
933 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
934 err = -EINVAL;
935 goto err_unbind;
936 }
937
938 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
939
940 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
941 info->extack);
942 if (err)
943 goto err_unbind;
944 }
945
946 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
947 genlmsg_end(rsp, hdr);
948
949 err = genlmsg_reply(rsp, info);
950 if (err)
951 goto err_unbind;
952
953 netdev_unlock(netdev);
954
955 mutex_unlock(&priv->lock);
956
957 return 0;
958
959 err_unbind:
960 net_devmem_unbind_dmabuf(binding);
961 err_unlock:
962 netdev_unlock(netdev);
963 err_unlock_sock:
964 mutex_unlock(&priv->lock);
965 err_genlmsg_free:
966 nlmsg_free(rsp);
967 return err;
968 }
969
netdev_nl_bind_tx_doit(struct sk_buff * skb,struct genl_info * info)970 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info)
971 {
972 struct net_devmem_dmabuf_binding *binding;
973 struct netdev_nl_sock *priv;
974 struct net_device *netdev;
975 u32 ifindex, dmabuf_fd;
976 struct sk_buff *rsp;
977 int err = 0;
978 void *hdr;
979
980 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
981 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD))
982 return -EINVAL;
983
984 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
985 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
986
987 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
988 if (IS_ERR(priv))
989 return PTR_ERR(priv);
990
991 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
992 if (!rsp)
993 return -ENOMEM;
994
995 hdr = genlmsg_iput(rsp, info);
996 if (!hdr) {
997 err = -EMSGSIZE;
998 goto err_genlmsg_free;
999 }
1000
1001 mutex_lock(&priv->lock);
1002
1003 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
1004 if (!netdev) {
1005 err = -ENODEV;
1006 goto err_unlock_sock;
1007 }
1008
1009 if (!netif_device_present(netdev)) {
1010 err = -ENODEV;
1011 goto err_unlock_netdev;
1012 }
1013
1014 if (!netdev->netmem_tx) {
1015 err = -EOPNOTSUPP;
1016 NL_SET_ERR_MSG(info->extack,
1017 "Driver does not support netmem TX");
1018 goto err_unlock_netdev;
1019 }
1020
1021 binding = net_devmem_bind_dmabuf(netdev, DMA_TO_DEVICE, dmabuf_fd, priv,
1022 info->extack);
1023 if (IS_ERR(binding)) {
1024 err = PTR_ERR(binding);
1025 goto err_unlock_netdev;
1026 }
1027
1028 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
1029 genlmsg_end(rsp, hdr);
1030
1031 netdev_unlock(netdev);
1032 mutex_unlock(&priv->lock);
1033
1034 return genlmsg_reply(rsp, info);
1035
1036 err_unlock_netdev:
1037 netdev_unlock(netdev);
1038 err_unlock_sock:
1039 mutex_unlock(&priv->lock);
1040 err_genlmsg_free:
1041 nlmsg_free(rsp);
1042 return err;
1043 }
1044
netdev_nl_sock_priv_init(struct netdev_nl_sock * priv)1045 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv)
1046 {
1047 INIT_LIST_HEAD(&priv->bindings);
1048 mutex_init(&priv->lock);
1049 }
1050
netdev_nl_sock_priv_destroy(struct netdev_nl_sock * priv)1051 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv)
1052 {
1053 struct net_devmem_dmabuf_binding *binding;
1054 struct net_devmem_dmabuf_binding *temp;
1055 netdevice_tracker dev_tracker;
1056 struct net_device *dev;
1057
1058 mutex_lock(&priv->lock);
1059 list_for_each_entry_safe(binding, temp, &priv->bindings, list) {
1060 mutex_lock(&binding->lock);
1061 dev = binding->dev;
1062 if (!dev) {
1063 mutex_unlock(&binding->lock);
1064 net_devmem_unbind_dmabuf(binding);
1065 continue;
1066 }
1067 netdev_hold(dev, &dev_tracker, GFP_KERNEL);
1068 mutex_unlock(&binding->lock);
1069
1070 netdev_lock(dev);
1071 net_devmem_unbind_dmabuf(binding);
1072 netdev_unlock(dev);
1073 netdev_put(dev, &dev_tracker);
1074 }
1075 mutex_unlock(&priv->lock);
1076 }
1077
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)1078 static int netdev_genl_netdevice_event(struct notifier_block *nb,
1079 unsigned long event, void *ptr)
1080 {
1081 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
1082
1083 switch (event) {
1084 case NETDEV_REGISTER:
1085 netdev_lock_ops_to_full(netdev);
1086 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
1087 netdev_unlock_full_to_ops(netdev);
1088 break;
1089 case NETDEV_UNREGISTER:
1090 netdev_lock(netdev);
1091 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
1092 netdev_unlock(netdev);
1093 break;
1094 case NETDEV_XDP_FEAT_CHANGE:
1095 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
1096 break;
1097 }
1098
1099 return NOTIFY_OK;
1100 }
1101
1102 static struct notifier_block netdev_genl_nb = {
1103 .notifier_call = netdev_genl_netdevice_event,
1104 };
1105
netdev_genl_init(void)1106 static int __init netdev_genl_init(void)
1107 {
1108 int err;
1109
1110 err = register_netdevice_notifier(&netdev_genl_nb);
1111 if (err)
1112 return err;
1113
1114 err = genl_register_family(&netdev_nl_family);
1115 if (err)
1116 goto err_unreg_ntf;
1117
1118 return 0;
1119
1120 err_unreg_ntf:
1121 unregister_netdevice_notifier(&netdev_genl_nb);
1122 return err;
1123 }
1124
1125 subsys_initcall(netdev_genl_init);
1126