1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/rtnetlink.h>
6 #include <net/busy_poll.h>
7 #include <net/net_namespace.h>
8 #include <net/netdev_queues.h>
9 #include <net/netdev_rx_queue.h>
10 #include <net/sock.h>
11 #include <net/xdp.h>
12 #include <net/xdp_sock.h>
13
14 #include "dev.h"
15 #include "devmem.h"
16 #include "netdev-genl-gen.h"
17
18 struct netdev_nl_dump_ctx {
19 unsigned long ifindex;
20 unsigned int rxq_idx;
21 unsigned int txq_idx;
22 unsigned int napi_id;
23 };
24
netdev_dump_ctx(struct netlink_callback * cb)25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
26 {
27 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
28
29 return (struct netdev_nl_dump_ctx *)cb->ctx;
30 }
31
32 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
34 const struct genl_info *info)
35 {
36 u64 xsk_features = 0;
37 u64 xdp_rx_meta = 0;
38 void *hdr;
39
40 hdr = genlmsg_iput(rsp, info);
41 if (!hdr)
42 return -EMSGSIZE;
43
44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
45 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
46 xdp_rx_meta |= flag;
47 XDP_METADATA_KFUNC_xxx
48 #undef XDP_METADATA_KFUNC
49
50 if (netdev->xsk_tx_metadata_ops) {
51 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
52 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
53 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
54 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
55 }
56
57 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
58 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
59 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
60 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
61 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
62 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
63 xsk_features, NETDEV_A_DEV_PAD))
64 goto err_cancel_msg;
65
66 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
67 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
68 netdev->xdp_zc_max_segs))
69 goto err_cancel_msg;
70 }
71
72 genlmsg_end(rsp, hdr);
73
74 return 0;
75
76 err_cancel_msg:
77 genlmsg_cancel(rsp, hdr);
78 return -EMSGSIZE;
79 }
80
81 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)82 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
83 {
84 struct genl_info info;
85 struct sk_buff *ntf;
86
87 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
88 NETDEV_NLGRP_MGMT))
89 return;
90
91 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
92
93 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
94 if (!ntf)
95 return;
96
97 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
98 nlmsg_free(ntf);
99 return;
100 }
101
102 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
103 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
104 }
105
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)106 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
107 {
108 struct net_device *netdev;
109 struct sk_buff *rsp;
110 u32 ifindex;
111 int err;
112
113 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
114 return -EINVAL;
115
116 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
117
118 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
119 if (!rsp)
120 return -ENOMEM;
121
122 rtnl_lock();
123
124 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
125 if (netdev)
126 err = netdev_nl_dev_fill(netdev, rsp, info);
127 else
128 err = -ENODEV;
129
130 rtnl_unlock();
131
132 if (err)
133 goto err_free_msg;
134
135 return genlmsg_reply(rsp, info);
136
137 err_free_msg:
138 nlmsg_free(rsp);
139 return err;
140 }
141
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)142 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
143 {
144 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
145 struct net *net = sock_net(skb->sk);
146 struct net_device *netdev;
147 int err = 0;
148
149 rtnl_lock();
150 for_each_netdev_dump(net, netdev, ctx->ifindex) {
151 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
152 if (err < 0)
153 break;
154 }
155 rtnl_unlock();
156
157 return err;
158 }
159
160 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)161 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
162 const struct genl_info *info)
163 {
164 unsigned long irq_suspend_timeout;
165 unsigned long gro_flush_timeout;
166 u32 napi_defer_hard_irqs;
167 void *hdr;
168 pid_t pid;
169
170 if (WARN_ON_ONCE(!napi->dev))
171 return -EINVAL;
172 if (!(napi->dev->flags & IFF_UP))
173 return 0;
174
175 hdr = genlmsg_iput(rsp, info);
176 if (!hdr)
177 return -EMSGSIZE;
178
179 if (napi->napi_id >= MIN_NAPI_ID &&
180 nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
181 goto nla_put_failure;
182
183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
184 goto nla_put_failure;
185
186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
187 goto nla_put_failure;
188
189 if (napi->thread) {
190 pid = task_pid_nr(napi->thread);
191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
192 goto nla_put_failure;
193 }
194
195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
197 napi_defer_hard_irqs))
198 goto nla_put_failure;
199
200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
202 irq_suspend_timeout))
203 goto nla_put_failure;
204
205 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
207 gro_flush_timeout))
208 goto nla_put_failure;
209
210 genlmsg_end(rsp, hdr);
211
212 return 0;
213
214 nla_put_failure:
215 genlmsg_cancel(rsp, hdr);
216 return -EMSGSIZE;
217 }
218
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)219 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
220 {
221 struct napi_struct *napi;
222 struct sk_buff *rsp;
223 u32 napi_id;
224 int err;
225
226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
227 return -EINVAL;
228
229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
230
231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
232 if (!rsp)
233 return -ENOMEM;
234
235 rtnl_lock();
236 rcu_read_lock();
237
238 napi = napi_by_id(napi_id);
239 if (napi) {
240 err = netdev_nl_napi_fill_one(rsp, napi, info);
241 } else {
242 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
243 err = -ENOENT;
244 }
245
246 rcu_read_unlock();
247 rtnl_unlock();
248
249 if (err)
250 goto err_free_msg;
251
252 return genlmsg_reply(rsp, info);
253
254 err_free_msg:
255 nlmsg_free(rsp);
256 return err;
257 }
258
259 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)260 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
261 const struct genl_info *info,
262 struct netdev_nl_dump_ctx *ctx)
263 {
264 struct napi_struct *napi;
265 int err = 0;
266
267 if (!(netdev->flags & IFF_UP))
268 return err;
269
270 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
271 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
272 continue;
273
274 err = netdev_nl_napi_fill_one(rsp, napi, info);
275 if (err)
276 return err;
277 ctx->napi_id = napi->napi_id;
278 }
279 return err;
280 }
281
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)282 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
283 {
284 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
285 const struct genl_info *info = genl_info_dump(cb);
286 struct net *net = sock_net(skb->sk);
287 struct net_device *netdev;
288 u32 ifindex = 0;
289 int err = 0;
290
291 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
292 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
293
294 rtnl_lock();
295 if (ifindex) {
296 netdev = __dev_get_by_index(net, ifindex);
297 if (netdev)
298 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
299 else
300 err = -ENODEV;
301 } else {
302 for_each_netdev_dump(net, netdev, ctx->ifindex) {
303 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
304 if (err < 0)
305 break;
306 ctx->napi_id = 0;
307 }
308 }
309 rtnl_unlock();
310
311 return err;
312 }
313
314 static int
netdev_nl_napi_set_config(struct napi_struct * napi,struct genl_info * info)315 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
316 {
317 u64 irq_suspend_timeout = 0;
318 u64 gro_flush_timeout = 0;
319 u32 defer = 0;
320
321 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
322 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
323 napi_set_defer_hard_irqs(napi, defer);
324 }
325
326 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
327 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
328 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
329 }
330
331 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
332 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
333 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
334 }
335
336 return 0;
337 }
338
netdev_nl_napi_set_doit(struct sk_buff * skb,struct genl_info * info)339 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
340 {
341 struct napi_struct *napi;
342 unsigned int napi_id;
343 int err;
344
345 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
346 return -EINVAL;
347
348 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
349
350 rtnl_lock();
351 rcu_read_lock();
352
353 napi = napi_by_id(napi_id);
354 if (napi) {
355 err = netdev_nl_napi_set_config(napi, info);
356 } else {
357 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
358 err = -ENOENT;
359 }
360
361 rcu_read_unlock();
362 rtnl_unlock();
363
364 return err;
365 }
366
367 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)368 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
369 u32 q_idx, u32 q_type, const struct genl_info *info)
370 {
371 struct net_devmem_dmabuf_binding *binding;
372 struct netdev_rx_queue *rxq;
373 struct netdev_queue *txq;
374 void *hdr;
375
376 hdr = genlmsg_iput(rsp, info);
377 if (!hdr)
378 return -EMSGSIZE;
379
380 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
381 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
382 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
383 goto nla_put_failure;
384
385 switch (q_type) {
386 case NETDEV_QUEUE_TYPE_RX:
387 rxq = __netif_get_rx_queue(netdev, q_idx);
388 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
389 rxq->napi->napi_id))
390 goto nla_put_failure;
391
392 binding = rxq->mp_params.mp_priv;
393 if (binding &&
394 nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id))
395 goto nla_put_failure;
396
397 break;
398 case NETDEV_QUEUE_TYPE_TX:
399 txq = netdev_get_tx_queue(netdev, q_idx);
400 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
401 txq->napi->napi_id))
402 goto nla_put_failure;
403 }
404
405 genlmsg_end(rsp, hdr);
406
407 return 0;
408
409 nla_put_failure:
410 genlmsg_cancel(rsp, hdr);
411 return -EMSGSIZE;
412 }
413
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)414 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
415 u32 q_type)
416 {
417 switch (q_type) {
418 case NETDEV_QUEUE_TYPE_RX:
419 if (q_id >= netdev->real_num_rx_queues)
420 return -EINVAL;
421 return 0;
422 case NETDEV_QUEUE_TYPE_TX:
423 if (q_id >= netdev->real_num_tx_queues)
424 return -EINVAL;
425 }
426 return 0;
427 }
428
429 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)430 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
431 u32 q_type, const struct genl_info *info)
432 {
433 int err;
434
435 if (!(netdev->flags & IFF_UP))
436 return -ENOENT;
437
438 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
439 if (err)
440 return err;
441
442 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
443 }
444
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)445 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
446 {
447 u32 q_id, q_type, ifindex;
448 struct net_device *netdev;
449 struct sk_buff *rsp;
450 int err;
451
452 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
453 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
454 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
455 return -EINVAL;
456
457 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
458 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
459 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
460
461 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
462 if (!rsp)
463 return -ENOMEM;
464
465 rtnl_lock();
466
467 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
468 if (netdev)
469 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
470 else
471 err = -ENODEV;
472
473 rtnl_unlock();
474
475 if (err)
476 goto err_free_msg;
477
478 return genlmsg_reply(rsp, info);
479
480 err_free_msg:
481 nlmsg_free(rsp);
482 return err;
483 }
484
485 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)486 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
487 const struct genl_info *info,
488 struct netdev_nl_dump_ctx *ctx)
489 {
490 int err = 0;
491
492 if (!(netdev->flags & IFF_UP))
493 return err;
494
495 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
496 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
497 NETDEV_QUEUE_TYPE_RX, info);
498 if (err)
499 return err;
500 }
501 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
502 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
503 NETDEV_QUEUE_TYPE_TX, info);
504 if (err)
505 return err;
506 }
507
508 return err;
509 }
510
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)511 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
512 {
513 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
514 const struct genl_info *info = genl_info_dump(cb);
515 struct net *net = sock_net(skb->sk);
516 struct net_device *netdev;
517 u32 ifindex = 0;
518 int err = 0;
519
520 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
521 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
522
523 rtnl_lock();
524 if (ifindex) {
525 netdev = __dev_get_by_index(net, ifindex);
526 if (netdev)
527 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
528 else
529 err = -ENODEV;
530 } else {
531 for_each_netdev_dump(net, netdev, ctx->ifindex) {
532 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
533 if (err < 0)
534 break;
535 ctx->rxq_idx = 0;
536 ctx->txq_idx = 0;
537 }
538 }
539 rtnl_unlock();
540
541 return err;
542 }
543
544 #define NETDEV_STAT_NOT_SET (~0ULL)
545
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)546 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
547 {
548 const u64 *add = _add;
549 u64 *sum = _sum;
550
551 while (size) {
552 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
553 *sum += *add;
554 sum++;
555 add++;
556 size -= 8;
557 }
558 }
559
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)560 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
561 {
562 if (value == NETDEV_STAT_NOT_SET)
563 return 0;
564 return nla_put_uint(rsp, attr_id, value);
565 }
566
567 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)568 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
569 {
570 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
571 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
572 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
573 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
574 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
575 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
576 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
577 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
578 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
579 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
580 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
581 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
582 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
583 return -EMSGSIZE;
584 return 0;
585 }
586
587 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)588 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
589 {
590 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
591 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
592 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
593 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
594 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
595 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
596 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
597 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
598 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
599 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
600 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
601 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
602 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
603 return -EMSGSIZE;
604 return 0;
605 }
606
607 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)608 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
609 u32 q_type, int i, const struct genl_info *info)
610 {
611 const struct netdev_stat_ops *ops = netdev->stat_ops;
612 struct netdev_queue_stats_rx rx;
613 struct netdev_queue_stats_tx tx;
614 void *hdr;
615
616 hdr = genlmsg_iput(rsp, info);
617 if (!hdr)
618 return -EMSGSIZE;
619 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
620 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
621 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
622 goto nla_put_failure;
623
624 switch (q_type) {
625 case NETDEV_QUEUE_TYPE_RX:
626 memset(&rx, 0xff, sizeof(rx));
627 ops->get_queue_stats_rx(netdev, i, &rx);
628 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
629 goto nla_cancel;
630 if (netdev_nl_stats_write_rx(rsp, &rx))
631 goto nla_put_failure;
632 break;
633 case NETDEV_QUEUE_TYPE_TX:
634 memset(&tx, 0xff, sizeof(tx));
635 ops->get_queue_stats_tx(netdev, i, &tx);
636 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
637 goto nla_cancel;
638 if (netdev_nl_stats_write_tx(rsp, &tx))
639 goto nla_put_failure;
640 break;
641 }
642
643 genlmsg_end(rsp, hdr);
644 return 0;
645
646 nla_cancel:
647 genlmsg_cancel(rsp, hdr);
648 return 0;
649 nla_put_failure:
650 genlmsg_cancel(rsp, hdr);
651 return -EMSGSIZE;
652 }
653
654 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)655 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
656 const struct genl_info *info,
657 struct netdev_nl_dump_ctx *ctx)
658 {
659 const struct netdev_stat_ops *ops = netdev->stat_ops;
660 int i, err;
661
662 if (!(netdev->flags & IFF_UP))
663 return 0;
664
665 i = ctx->rxq_idx;
666 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
667 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
668 i, info);
669 if (err)
670 return err;
671 ctx->rxq_idx = ++i;
672 }
673 i = ctx->txq_idx;
674 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
675 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
676 i, info);
677 if (err)
678 return err;
679 ctx->txq_idx = ++i;
680 }
681
682 ctx->rxq_idx = 0;
683 ctx->txq_idx = 0;
684 return 0;
685 }
686
687 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)688 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
689 const struct genl_info *info)
690 {
691 struct netdev_queue_stats_rx rx_sum, rx;
692 struct netdev_queue_stats_tx tx_sum, tx;
693 const struct netdev_stat_ops *ops;
694 void *hdr;
695 int i;
696
697 ops = netdev->stat_ops;
698 /* Netdev can't guarantee any complete counters */
699 if (!ops->get_base_stats)
700 return 0;
701
702 memset(&rx_sum, 0xff, sizeof(rx_sum));
703 memset(&tx_sum, 0xff, sizeof(tx_sum));
704
705 ops->get_base_stats(netdev, &rx_sum, &tx_sum);
706
707 /* The op was there, but nothing reported, don't bother */
708 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
709 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
710 return 0;
711
712 hdr = genlmsg_iput(rsp, info);
713 if (!hdr)
714 return -EMSGSIZE;
715 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
716 goto nla_put_failure;
717
718 for (i = 0; i < netdev->real_num_rx_queues; i++) {
719 memset(&rx, 0xff, sizeof(rx));
720 if (ops->get_queue_stats_rx)
721 ops->get_queue_stats_rx(netdev, i, &rx);
722 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx));
723 }
724 for (i = 0; i < netdev->real_num_tx_queues; i++) {
725 memset(&tx, 0xff, sizeof(tx));
726 if (ops->get_queue_stats_tx)
727 ops->get_queue_stats_tx(netdev, i, &tx);
728 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx));
729 }
730
731 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
732 netdev_nl_stats_write_tx(rsp, &tx_sum))
733 goto nla_put_failure;
734
735 genlmsg_end(rsp, hdr);
736 return 0;
737
738 nla_put_failure:
739 genlmsg_cancel(rsp, hdr);
740 return -EMSGSIZE;
741 }
742
743 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)744 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
745 struct sk_buff *skb, const struct genl_info *info,
746 struct netdev_nl_dump_ctx *ctx)
747 {
748 if (!netdev->stat_ops)
749 return 0;
750
751 switch (scope) {
752 case 0:
753 return netdev_nl_stats_by_netdev(netdev, skb, info);
754 case NETDEV_QSTATS_SCOPE_QUEUE:
755 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
756 }
757
758 return -EINVAL; /* Should not happen, per netlink policy */
759 }
760
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)761 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
762 struct netlink_callback *cb)
763 {
764 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
765 const struct genl_info *info = genl_info_dump(cb);
766 struct net *net = sock_net(skb->sk);
767 struct net_device *netdev;
768 unsigned int ifindex;
769 unsigned int scope;
770 int err = 0;
771
772 scope = 0;
773 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
774 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
775
776 ifindex = 0;
777 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
778 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
779
780 rtnl_lock();
781 if (ifindex) {
782 netdev = __dev_get_by_index(net, ifindex);
783 if (netdev && netdev->stat_ops) {
784 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
785 info, ctx);
786 } else {
787 NL_SET_BAD_ATTR(info->extack,
788 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
789 err = netdev ? -EOPNOTSUPP : -ENODEV;
790 }
791 } else {
792 for_each_netdev_dump(net, netdev, ctx->ifindex) {
793 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
794 info, ctx);
795 if (err < 0)
796 break;
797 }
798 }
799 rtnl_unlock();
800
801 return err;
802 }
803
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)804 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
805 {
806 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
807 struct net_devmem_dmabuf_binding *binding;
808 struct list_head *sock_binding_list;
809 u32 ifindex, dmabuf_fd, rxq_idx;
810 struct net_device *netdev;
811 struct sk_buff *rsp;
812 struct nlattr *attr;
813 int rem, err = 0;
814 void *hdr;
815
816 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
817 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
818 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
819 return -EINVAL;
820
821 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
822 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
823
824 sock_binding_list = genl_sk_priv_get(&netdev_nl_family,
825 NETLINK_CB(skb).sk);
826 if (IS_ERR(sock_binding_list))
827 return PTR_ERR(sock_binding_list);
828
829 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
830 if (!rsp)
831 return -ENOMEM;
832
833 hdr = genlmsg_iput(rsp, info);
834 if (!hdr) {
835 err = -EMSGSIZE;
836 goto err_genlmsg_free;
837 }
838
839 rtnl_lock();
840
841 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
842 if (!netdev || !netif_device_present(netdev)) {
843 err = -ENODEV;
844 goto err_unlock;
845 }
846
847 if (dev_xdp_prog_count(netdev)) {
848 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached");
849 err = -EEXIST;
850 goto err_unlock;
851 }
852
853 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
854 if (IS_ERR(binding)) {
855 err = PTR_ERR(binding);
856 goto err_unlock;
857 }
858
859 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
860 genlmsg_data(info->genlhdr),
861 genlmsg_len(info->genlhdr), rem) {
862 err = nla_parse_nested(
863 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
864 netdev_queue_id_nl_policy, info->extack);
865 if (err < 0)
866 goto err_unbind;
867
868 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
869 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
870 err = -EINVAL;
871 goto err_unbind;
872 }
873
874 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
875 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
876 err = -EINVAL;
877 goto err_unbind;
878 }
879
880 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
881
882 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
883 info->extack);
884 if (err)
885 goto err_unbind;
886 }
887
888 list_add(&binding->list, sock_binding_list);
889
890 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
891 genlmsg_end(rsp, hdr);
892
893 err = genlmsg_reply(rsp, info);
894 if (err)
895 goto err_unbind;
896
897 rtnl_unlock();
898
899 return 0;
900
901 err_unbind:
902 net_devmem_unbind_dmabuf(binding);
903 err_unlock:
904 rtnl_unlock();
905 err_genlmsg_free:
906 nlmsg_free(rsp);
907 return err;
908 }
909
netdev_nl_sock_priv_init(struct list_head * priv)910 void netdev_nl_sock_priv_init(struct list_head *priv)
911 {
912 INIT_LIST_HEAD(priv);
913 }
914
netdev_nl_sock_priv_destroy(struct list_head * priv)915 void netdev_nl_sock_priv_destroy(struct list_head *priv)
916 {
917 struct net_devmem_dmabuf_binding *binding;
918 struct net_devmem_dmabuf_binding *temp;
919
920 list_for_each_entry_safe(binding, temp, priv, list) {
921 rtnl_lock();
922 net_devmem_unbind_dmabuf(binding);
923 rtnl_unlock();
924 }
925 }
926
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)927 static int netdev_genl_netdevice_event(struct notifier_block *nb,
928 unsigned long event, void *ptr)
929 {
930 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
931
932 switch (event) {
933 case NETDEV_REGISTER:
934 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
935 break;
936 case NETDEV_UNREGISTER:
937 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
938 break;
939 case NETDEV_XDP_FEAT_CHANGE:
940 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
941 break;
942 }
943
944 return NOTIFY_OK;
945 }
946
947 static struct notifier_block netdev_genl_nb = {
948 .notifier_call = netdev_genl_netdevice_event,
949 };
950
netdev_genl_init(void)951 static int __init netdev_genl_init(void)
952 {
953 int err;
954
955 err = register_netdevice_notifier(&netdev_genl_nb);
956 if (err)
957 return err;
958
959 err = genl_register_family(&netdev_nl_family);
960 if (err)
961 goto err_unreg_ntf;
962
963 return 0;
964
965 err_unreg_ntf:
966 unregister_netdevice_notifier(&netdev_genl_nb);
967 return err;
968 }
969
970 subsys_initcall(netdev_genl_init);
971