1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/rtnetlink.h>
6 #include <net/busy_poll.h>
7 #include <net/net_namespace.h>
8 #include <net/netdev_queues.h>
9 #include <net/netdev_rx_queue.h>
10 #include <net/sock.h>
11 #include <net/xdp.h>
12 #include <net/xdp_sock.h>
13 #include <net/page_pool/memory_provider.h>
14
15 #include "dev.h"
16 #include "devmem.h"
17 #include "netdev-genl-gen.h"
18
19 struct netdev_nl_dump_ctx {
20 unsigned long ifindex;
21 unsigned int rxq_idx;
22 unsigned int txq_idx;
23 unsigned int napi_id;
24 };
25
netdev_dump_ctx(struct netlink_callback * cb)26 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
27 {
28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
29
30 return (struct netdev_nl_dump_ctx *)cb->ctx;
31 }
32
33 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)34 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
35 const struct genl_info *info)
36 {
37 u64 xsk_features = 0;
38 u64 xdp_rx_meta = 0;
39 void *hdr;
40
41 hdr = genlmsg_iput(rsp, info);
42 if (!hdr)
43 return -EMSGSIZE;
44
45 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
46 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
47 xdp_rx_meta |= flag;
48 XDP_METADATA_KFUNC_xxx
49 #undef XDP_METADATA_KFUNC
50
51 if (netdev->xsk_tx_metadata_ops) {
52 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
53 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
54 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
55 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
56 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time)
57 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO;
58 }
59
60 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
61 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
62 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
64 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
65 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
66 xsk_features, NETDEV_A_DEV_PAD))
67 goto err_cancel_msg;
68
69 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
70 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
71 netdev->xdp_zc_max_segs))
72 goto err_cancel_msg;
73 }
74
75 genlmsg_end(rsp, hdr);
76
77 return 0;
78
79 err_cancel_msg:
80 genlmsg_cancel(rsp, hdr);
81 return -EMSGSIZE;
82 }
83
84 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)85 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
86 {
87 struct genl_info info;
88 struct sk_buff *ntf;
89
90 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
91 NETDEV_NLGRP_MGMT))
92 return;
93
94 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
95
96 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
97 if (!ntf)
98 return;
99
100 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
101 nlmsg_free(ntf);
102 return;
103 }
104
105 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
106 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
107 }
108
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)109 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
110 {
111 struct net_device *netdev;
112 struct sk_buff *rsp;
113 u32 ifindex;
114 int err;
115
116 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
117 return -EINVAL;
118
119 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
120
121 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
122 if (!rsp)
123 return -ENOMEM;
124
125 rtnl_lock();
126
127 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
128 if (netdev)
129 err = netdev_nl_dev_fill(netdev, rsp, info);
130 else
131 err = -ENODEV;
132
133 rtnl_unlock();
134
135 if (err)
136 goto err_free_msg;
137
138 return genlmsg_reply(rsp, info);
139
140 err_free_msg:
141 nlmsg_free(rsp);
142 return err;
143 }
144
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)145 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
146 {
147 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
148 struct net *net = sock_net(skb->sk);
149 struct net_device *netdev;
150 int err = 0;
151
152 rtnl_lock();
153 for_each_netdev_dump(net, netdev, ctx->ifindex) {
154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
155 if (err < 0)
156 break;
157 }
158 rtnl_unlock();
159
160 return err;
161 }
162
163 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)164 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
165 const struct genl_info *info)
166 {
167 unsigned long irq_suspend_timeout;
168 unsigned long gro_flush_timeout;
169 u32 napi_defer_hard_irqs;
170 void *hdr;
171 pid_t pid;
172
173 if (!napi->dev->up)
174 return 0;
175
176 hdr = genlmsg_iput(rsp, info);
177 if (!hdr)
178 return -EMSGSIZE;
179
180 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
181 goto nla_put_failure;
182
183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
184 goto nla_put_failure;
185
186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
187 goto nla_put_failure;
188
189 if (napi->thread) {
190 pid = task_pid_nr(napi->thread);
191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
192 goto nla_put_failure;
193 }
194
195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
197 napi_defer_hard_irqs))
198 goto nla_put_failure;
199
200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
202 irq_suspend_timeout))
203 goto nla_put_failure;
204
205 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
207 gro_flush_timeout))
208 goto nla_put_failure;
209
210 genlmsg_end(rsp, hdr);
211
212 return 0;
213
214 nla_put_failure:
215 genlmsg_cancel(rsp, hdr);
216 return -EMSGSIZE;
217 }
218
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)219 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
220 {
221 struct napi_struct *napi;
222 struct sk_buff *rsp;
223 u32 napi_id;
224 int err;
225
226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
227 return -EINVAL;
228
229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
230
231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
232 if (!rsp)
233 return -ENOMEM;
234
235 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
236 if (napi) {
237 err = netdev_nl_napi_fill_one(rsp, napi, info);
238 netdev_unlock(napi->dev);
239 } else {
240 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
241 err = -ENOENT;
242 }
243
244 if (err) {
245 goto err_free_msg;
246 } else if (!rsp->len) {
247 err = -ENOENT;
248 goto err_free_msg;
249 }
250
251 return genlmsg_reply(rsp, info);
252
253 err_free_msg:
254 nlmsg_free(rsp);
255 return err;
256 }
257
258 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)259 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
260 const struct genl_info *info,
261 struct netdev_nl_dump_ctx *ctx)
262 {
263 struct napi_struct *napi;
264 unsigned int prev_id;
265 int err = 0;
266
267 if (!netdev->up)
268 return err;
269
270 prev_id = UINT_MAX;
271 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
272 if (!napi_id_valid(napi->napi_id))
273 continue;
274
275 /* Dump continuation below depends on the list being sorted */
276 WARN_ON_ONCE(napi->napi_id >= prev_id);
277 prev_id = napi->napi_id;
278
279 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
280 continue;
281
282 err = netdev_nl_napi_fill_one(rsp, napi, info);
283 if (err)
284 return err;
285 ctx->napi_id = napi->napi_id;
286 }
287 return err;
288 }
289
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)290 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
291 {
292 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
293 const struct genl_info *info = genl_info_dump(cb);
294 struct net *net = sock_net(skb->sk);
295 struct net_device *netdev;
296 u32 ifindex = 0;
297 int err = 0;
298
299 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
300 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
301
302 if (ifindex) {
303 netdev = netdev_get_by_index_lock(net, ifindex);
304 if (netdev) {
305 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
306 netdev_unlock(netdev);
307 } else {
308 err = -ENODEV;
309 }
310 } else {
311 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
312 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
313 if (err < 0)
314 break;
315 ctx->napi_id = 0;
316 }
317 }
318
319 return err;
320 }
321
322 static int
netdev_nl_napi_set_config(struct napi_struct * napi,struct genl_info * info)323 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
324 {
325 u64 irq_suspend_timeout = 0;
326 u64 gro_flush_timeout = 0;
327 u32 defer = 0;
328
329 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
330 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
331 napi_set_defer_hard_irqs(napi, defer);
332 }
333
334 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
335 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
336 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
337 }
338
339 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
340 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
341 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
342 }
343
344 return 0;
345 }
346
netdev_nl_napi_set_doit(struct sk_buff * skb,struct genl_info * info)347 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
348 {
349 struct napi_struct *napi;
350 unsigned int napi_id;
351 int err;
352
353 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
354 return -EINVAL;
355
356 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
357
358 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
359 if (napi) {
360 err = netdev_nl_napi_set_config(napi, info);
361 netdev_unlock(napi->dev);
362 } else {
363 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
364 err = -ENOENT;
365 }
366
367 return err;
368 }
369
nla_put_napi_id(struct sk_buff * skb,const struct napi_struct * napi)370 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi)
371 {
372 if (napi && napi_id_valid(napi->napi_id))
373 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id);
374 return 0;
375 }
376
377 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)378 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
379 u32 q_idx, u32 q_type, const struct genl_info *info)
380 {
381 struct pp_memory_provider_params *params;
382 struct netdev_rx_queue *rxq;
383 struct netdev_queue *txq;
384 void *hdr;
385
386 hdr = genlmsg_iput(rsp, info);
387 if (!hdr)
388 return -EMSGSIZE;
389
390 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
391 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
392 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
393 goto nla_put_failure;
394
395 switch (q_type) {
396 case NETDEV_QUEUE_TYPE_RX:
397 rxq = __netif_get_rx_queue(netdev, q_idx);
398 if (nla_put_napi_id(rsp, rxq->napi))
399 goto nla_put_failure;
400
401 params = &rxq->mp_params;
402 if (params->mp_ops &&
403 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq))
404 goto nla_put_failure;
405 #ifdef CONFIG_XDP_SOCKETS
406 if (rxq->pool)
407 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
408 goto nla_put_failure;
409 #endif
410
411 break;
412 case NETDEV_QUEUE_TYPE_TX:
413 txq = netdev_get_tx_queue(netdev, q_idx);
414 if (nla_put_napi_id(rsp, txq->napi))
415 goto nla_put_failure;
416 #ifdef CONFIG_XDP_SOCKETS
417 if (txq->pool)
418 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
419 goto nla_put_failure;
420 #endif
421 break;
422 }
423
424 genlmsg_end(rsp, hdr);
425
426 return 0;
427
428 nla_put_failure:
429 genlmsg_cancel(rsp, hdr);
430 return -EMSGSIZE;
431 }
432
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)433 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
434 u32 q_type)
435 {
436 switch (q_type) {
437 case NETDEV_QUEUE_TYPE_RX:
438 if (q_id >= netdev->real_num_rx_queues)
439 return -EINVAL;
440 return 0;
441 case NETDEV_QUEUE_TYPE_TX:
442 if (q_id >= netdev->real_num_tx_queues)
443 return -EINVAL;
444 }
445 return 0;
446 }
447
448 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)449 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
450 u32 q_type, const struct genl_info *info)
451 {
452 int err;
453
454 if (!netdev->up)
455 return -ENOENT;
456
457 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
458 if (err)
459 return err;
460
461 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
462 }
463
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)464 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
465 {
466 u32 q_id, q_type, ifindex;
467 struct net_device *netdev;
468 struct sk_buff *rsp;
469 int err;
470
471 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
472 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
473 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
474 return -EINVAL;
475
476 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
477 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
478 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
479
480 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
481 if (!rsp)
482 return -ENOMEM;
483
484 rtnl_lock();
485
486 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
487 if (netdev) {
488 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
489 netdev_unlock(netdev);
490 } else {
491 err = -ENODEV;
492 }
493
494 rtnl_unlock();
495
496 if (err)
497 goto err_free_msg;
498
499 return genlmsg_reply(rsp, info);
500
501 err_free_msg:
502 nlmsg_free(rsp);
503 return err;
504 }
505
506 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)507 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
508 const struct genl_info *info,
509 struct netdev_nl_dump_ctx *ctx)
510 {
511 int err = 0;
512
513 if (!netdev->up)
514 return err;
515
516 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
517 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
518 NETDEV_QUEUE_TYPE_RX, info);
519 if (err)
520 return err;
521 }
522 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
523 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
524 NETDEV_QUEUE_TYPE_TX, info);
525 if (err)
526 return err;
527 }
528
529 return err;
530 }
531
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)532 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
533 {
534 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
535 const struct genl_info *info = genl_info_dump(cb);
536 struct net *net = sock_net(skb->sk);
537 struct net_device *netdev;
538 u32 ifindex = 0;
539 int err = 0;
540
541 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
542 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
543
544 rtnl_lock();
545 if (ifindex) {
546 netdev = netdev_get_by_index_lock(net, ifindex);
547 if (netdev) {
548 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
549 netdev_unlock(netdev);
550 } else {
551 err = -ENODEV;
552 }
553 } else {
554 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
555 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
556 if (err < 0)
557 break;
558 ctx->rxq_idx = 0;
559 ctx->txq_idx = 0;
560 }
561 }
562 rtnl_unlock();
563
564 return err;
565 }
566
567 #define NETDEV_STAT_NOT_SET (~0ULL)
568
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)569 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
570 {
571 const u64 *add = _add;
572 u64 *sum = _sum;
573
574 while (size) {
575 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
576 *sum += *add;
577 sum++;
578 add++;
579 size -= 8;
580 }
581 }
582
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)583 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
584 {
585 if (value == NETDEV_STAT_NOT_SET)
586 return 0;
587 return nla_put_uint(rsp, attr_id, value);
588 }
589
590 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)591 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
592 {
593 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
594 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
595 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
596 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
597 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
598 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) ||
599 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
600 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
601 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
602 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
603 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
604 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
605 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
606 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
607 return -EMSGSIZE;
608 return 0;
609 }
610
611 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)612 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
613 {
614 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
615 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
616 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
617 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
618 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
619 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
620 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
621 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
622 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
623 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
624 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
625 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
626 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
627 return -EMSGSIZE;
628 return 0;
629 }
630
631 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)632 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
633 u32 q_type, int i, const struct genl_info *info)
634 {
635 const struct netdev_stat_ops *ops = netdev->stat_ops;
636 struct netdev_queue_stats_rx rx;
637 struct netdev_queue_stats_tx tx;
638 void *hdr;
639
640 hdr = genlmsg_iput(rsp, info);
641 if (!hdr)
642 return -EMSGSIZE;
643 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
644 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
645 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
646 goto nla_put_failure;
647
648 switch (q_type) {
649 case NETDEV_QUEUE_TYPE_RX:
650 memset(&rx, 0xff, sizeof(rx));
651 ops->get_queue_stats_rx(netdev, i, &rx);
652 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
653 goto nla_cancel;
654 if (netdev_nl_stats_write_rx(rsp, &rx))
655 goto nla_put_failure;
656 break;
657 case NETDEV_QUEUE_TYPE_TX:
658 memset(&tx, 0xff, sizeof(tx));
659 ops->get_queue_stats_tx(netdev, i, &tx);
660 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
661 goto nla_cancel;
662 if (netdev_nl_stats_write_tx(rsp, &tx))
663 goto nla_put_failure;
664 break;
665 }
666
667 genlmsg_end(rsp, hdr);
668 return 0;
669
670 nla_cancel:
671 genlmsg_cancel(rsp, hdr);
672 return 0;
673 nla_put_failure:
674 genlmsg_cancel(rsp, hdr);
675 return -EMSGSIZE;
676 }
677
678 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)679 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
680 const struct genl_info *info,
681 struct netdev_nl_dump_ctx *ctx)
682 {
683 const struct netdev_stat_ops *ops = netdev->stat_ops;
684 int i, err;
685
686 if (!(netdev->flags & IFF_UP))
687 return 0;
688
689 i = ctx->rxq_idx;
690 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
691 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
692 i, info);
693 if (err)
694 return err;
695 ctx->rxq_idx = ++i;
696 }
697 i = ctx->txq_idx;
698 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
699 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
700 i, info);
701 if (err)
702 return err;
703 ctx->txq_idx = ++i;
704 }
705
706 ctx->rxq_idx = 0;
707 ctx->txq_idx = 0;
708 return 0;
709 }
710
711 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)712 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
713 const struct genl_info *info)
714 {
715 struct netdev_queue_stats_rx rx_sum, rx;
716 struct netdev_queue_stats_tx tx_sum, tx;
717 const struct netdev_stat_ops *ops;
718 void *hdr;
719 int i;
720
721 ops = netdev->stat_ops;
722 /* Netdev can't guarantee any complete counters */
723 if (!ops->get_base_stats)
724 return 0;
725
726 memset(&rx_sum, 0xff, sizeof(rx_sum));
727 memset(&tx_sum, 0xff, sizeof(tx_sum));
728
729 ops->get_base_stats(netdev, &rx_sum, &tx_sum);
730
731 /* The op was there, but nothing reported, don't bother */
732 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
733 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
734 return 0;
735
736 hdr = genlmsg_iput(rsp, info);
737 if (!hdr)
738 return -EMSGSIZE;
739 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
740 goto nla_put_failure;
741
742 for (i = 0; i < netdev->real_num_rx_queues; i++) {
743 memset(&rx, 0xff, sizeof(rx));
744 if (ops->get_queue_stats_rx)
745 ops->get_queue_stats_rx(netdev, i, &rx);
746 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx));
747 }
748 for (i = 0; i < netdev->real_num_tx_queues; i++) {
749 memset(&tx, 0xff, sizeof(tx));
750 if (ops->get_queue_stats_tx)
751 ops->get_queue_stats_tx(netdev, i, &tx);
752 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx));
753 }
754
755 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
756 netdev_nl_stats_write_tx(rsp, &tx_sum))
757 goto nla_put_failure;
758
759 genlmsg_end(rsp, hdr);
760 return 0;
761
762 nla_put_failure:
763 genlmsg_cancel(rsp, hdr);
764 return -EMSGSIZE;
765 }
766
767 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)768 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
769 struct sk_buff *skb, const struct genl_info *info,
770 struct netdev_nl_dump_ctx *ctx)
771 {
772 if (!netdev->stat_ops)
773 return 0;
774
775 switch (scope) {
776 case 0:
777 return netdev_nl_stats_by_netdev(netdev, skb, info);
778 case NETDEV_QSTATS_SCOPE_QUEUE:
779 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
780 }
781
782 return -EINVAL; /* Should not happen, per netlink policy */
783 }
784
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)785 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
786 struct netlink_callback *cb)
787 {
788 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
789 const struct genl_info *info = genl_info_dump(cb);
790 struct net *net = sock_net(skb->sk);
791 struct net_device *netdev;
792 unsigned int ifindex;
793 unsigned int scope;
794 int err = 0;
795
796 scope = 0;
797 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
798 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
799
800 ifindex = 0;
801 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
802 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
803
804 rtnl_lock();
805 if (ifindex) {
806 netdev = __dev_get_by_index(net, ifindex);
807 if (netdev && netdev->stat_ops) {
808 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
809 info, ctx);
810 } else {
811 NL_SET_BAD_ATTR(info->extack,
812 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
813 err = netdev ? -EOPNOTSUPP : -ENODEV;
814 }
815 } else {
816 for_each_netdev_dump(net, netdev, ctx->ifindex) {
817 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
818 info, ctx);
819 if (err < 0)
820 break;
821 }
822 }
823 rtnl_unlock();
824
825 return err;
826 }
827
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)828 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
829 {
830 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
831 struct net_devmem_dmabuf_binding *binding;
832 u32 ifindex, dmabuf_fd, rxq_idx;
833 struct netdev_nl_sock *priv;
834 struct net_device *netdev;
835 struct sk_buff *rsp;
836 struct nlattr *attr;
837 int rem, err = 0;
838 void *hdr;
839
840 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
841 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
842 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
843 return -EINVAL;
844
845 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
846 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
847
848 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
849 if (IS_ERR(priv))
850 return PTR_ERR(priv);
851
852 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
853 if (!rsp)
854 return -ENOMEM;
855
856 hdr = genlmsg_iput(rsp, info);
857 if (!hdr) {
858 err = -EMSGSIZE;
859 goto err_genlmsg_free;
860 }
861
862 mutex_lock(&priv->lock);
863
864 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
865 if (!netdev || !netif_device_present(netdev)) {
866 err = -ENODEV;
867 goto err_unlock_sock;
868 }
869
870 if (!netdev_need_ops_lock(netdev)) {
871 err = -EOPNOTSUPP;
872 NL_SET_BAD_ATTR(info->extack,
873 info->attrs[NETDEV_A_DEV_IFINDEX]);
874 goto err_unlock;
875 }
876
877 if (dev_xdp_prog_count(netdev)) {
878 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached");
879 err = -EEXIST;
880 goto err_unlock;
881 }
882
883 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
884 if (IS_ERR(binding)) {
885 err = PTR_ERR(binding);
886 goto err_unlock;
887 }
888
889 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
890 genlmsg_data(info->genlhdr),
891 genlmsg_len(info->genlhdr), rem) {
892 err = nla_parse_nested(
893 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
894 netdev_queue_id_nl_policy, info->extack);
895 if (err < 0)
896 goto err_unbind;
897
898 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
899 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
900 err = -EINVAL;
901 goto err_unbind;
902 }
903
904 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
905 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
906 err = -EINVAL;
907 goto err_unbind;
908 }
909
910 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
911
912 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
913 info->extack);
914 if (err)
915 goto err_unbind;
916 }
917
918 list_add(&binding->list, &priv->bindings);
919
920 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
921 genlmsg_end(rsp, hdr);
922
923 err = genlmsg_reply(rsp, info);
924 if (err)
925 goto err_unbind;
926
927 netdev_unlock(netdev);
928
929 mutex_unlock(&priv->lock);
930
931 return 0;
932
933 err_unbind:
934 net_devmem_unbind_dmabuf(binding);
935 err_unlock:
936 netdev_unlock(netdev);
937 err_unlock_sock:
938 mutex_unlock(&priv->lock);
939 err_genlmsg_free:
940 nlmsg_free(rsp);
941 return err;
942 }
943
netdev_nl_sock_priv_init(struct netdev_nl_sock * priv)944 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv)
945 {
946 INIT_LIST_HEAD(&priv->bindings);
947 mutex_init(&priv->lock);
948 }
949
netdev_nl_sock_priv_destroy(struct netdev_nl_sock * priv)950 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv)
951 {
952 struct net_devmem_dmabuf_binding *binding;
953 struct net_devmem_dmabuf_binding *temp;
954
955 mutex_lock(&priv->lock);
956 list_for_each_entry_safe(binding, temp, &priv->bindings, list) {
957 netdev_lock(binding->dev);
958 net_devmem_unbind_dmabuf(binding);
959 netdev_unlock(binding->dev);
960 }
961 mutex_unlock(&priv->lock);
962 }
963
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)964 static int netdev_genl_netdevice_event(struct notifier_block *nb,
965 unsigned long event, void *ptr)
966 {
967 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
968
969 switch (event) {
970 case NETDEV_REGISTER:
971 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
972 break;
973 case NETDEV_UNREGISTER:
974 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
975 break;
976 case NETDEV_XDP_FEAT_CHANGE:
977 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
978 break;
979 }
980
981 return NOTIFY_OK;
982 }
983
984 static struct notifier_block netdev_genl_nb = {
985 .notifier_call = netdev_genl_netdevice_event,
986 };
987
netdev_genl_init(void)988 static int __init netdev_genl_init(void)
989 {
990 int err;
991
992 err = register_netdevice_notifier(&netdev_genl_nb);
993 if (err)
994 return err;
995
996 err = genl_register_family(&netdev_nl_family);
997 if (err)
998 goto err_unreg_ntf;
999
1000 return 0;
1001
1002 err_unreg_ntf:
1003 unregister_netdevice_notifier(&netdev_genl_nb);
1004 return err;
1005 }
1006
1007 subsys_initcall(netdev_genl_init);
1008