1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/rtnetlink.h>
6 #include <net/busy_poll.h>
7 #include <net/net_namespace.h>
8 #include <net/netdev_queues.h>
9 #include <net/netdev_rx_queue.h>
10 #include <net/sock.h>
11 #include <net/xdp.h>
12 #include <net/xdp_sock.h>
13
14 #include "dev.h"
15 #include "devmem.h"
16 #include "netdev-genl-gen.h"
17
18 struct netdev_nl_dump_ctx {
19 unsigned long ifindex;
20 unsigned int rxq_idx;
21 unsigned int txq_idx;
22 unsigned int napi_id;
23 };
24
netdev_dump_ctx(struct netlink_callback * cb)25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
26 {
27 NL_ASSERT_DUMP_CTX_FITS(struct netdev_nl_dump_ctx);
28
29 return (struct netdev_nl_dump_ctx *)cb->ctx;
30 }
31
32 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
34 const struct genl_info *info)
35 {
36 u64 xsk_features = 0;
37 u64 xdp_rx_meta = 0;
38 void *hdr;
39
40 hdr = genlmsg_iput(rsp, info);
41 if (!hdr)
42 return -EMSGSIZE;
43
44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
45 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
46 xdp_rx_meta |= flag;
47 XDP_METADATA_KFUNC_xxx
48 #undef XDP_METADATA_KFUNC
49
50 if (netdev->xsk_tx_metadata_ops) {
51 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
52 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
53 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
54 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
55 }
56
57 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
58 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
59 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
60 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
61 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
62 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
63 xsk_features, NETDEV_A_DEV_PAD))
64 goto err_cancel_msg;
65
66 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
67 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
68 netdev->xdp_zc_max_segs))
69 goto err_cancel_msg;
70 }
71
72 genlmsg_end(rsp, hdr);
73
74 return 0;
75
76 err_cancel_msg:
77 genlmsg_cancel(rsp, hdr);
78 return -EMSGSIZE;
79 }
80
81 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)82 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
83 {
84 struct genl_info info;
85 struct sk_buff *ntf;
86
87 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
88 NETDEV_NLGRP_MGMT))
89 return;
90
91 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
92
93 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
94 if (!ntf)
95 return;
96
97 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
98 nlmsg_free(ntf);
99 return;
100 }
101
102 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
103 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
104 }
105
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)106 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
107 {
108 struct net_device *netdev;
109 struct sk_buff *rsp;
110 u32 ifindex;
111 int err;
112
113 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
114 return -EINVAL;
115
116 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
117
118 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
119 if (!rsp)
120 return -ENOMEM;
121
122 rtnl_lock();
123
124 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
125 if (netdev)
126 err = netdev_nl_dev_fill(netdev, rsp, info);
127 else
128 err = -ENODEV;
129
130 rtnl_unlock();
131
132 if (err)
133 goto err_free_msg;
134
135 return genlmsg_reply(rsp, info);
136
137 err_free_msg:
138 nlmsg_free(rsp);
139 return err;
140 }
141
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)142 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
143 {
144 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
145 struct net *net = sock_net(skb->sk);
146 struct net_device *netdev;
147 int err = 0;
148
149 rtnl_lock();
150 for_each_netdev_dump(net, netdev, ctx->ifindex) {
151 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
152 if (err < 0)
153 break;
154 }
155 rtnl_unlock();
156
157 return err;
158 }
159
160 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)161 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
162 const struct genl_info *info)
163 {
164 void *hdr;
165 pid_t pid;
166
167 if (WARN_ON_ONCE(!napi->dev))
168 return -EINVAL;
169 if (!(napi->dev->flags & IFF_UP))
170 return 0;
171
172 hdr = genlmsg_iput(rsp, info);
173 if (!hdr)
174 return -EMSGSIZE;
175
176 if (napi->napi_id >= MIN_NAPI_ID &&
177 nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
178 goto nla_put_failure;
179
180 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
181 goto nla_put_failure;
182
183 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
184 goto nla_put_failure;
185
186 if (napi->thread) {
187 pid = task_pid_nr(napi->thread);
188 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
189 goto nla_put_failure;
190 }
191
192 genlmsg_end(rsp, hdr);
193
194 return 0;
195
196 nla_put_failure:
197 genlmsg_cancel(rsp, hdr);
198 return -EMSGSIZE;
199 }
200
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)201 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
202 {
203 struct napi_struct *napi;
204 struct sk_buff *rsp;
205 u32 napi_id;
206 int err;
207
208 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
209 return -EINVAL;
210
211 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
212
213 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
214 if (!rsp)
215 return -ENOMEM;
216
217 rtnl_lock();
218
219 napi = napi_by_id(napi_id);
220 if (napi) {
221 err = netdev_nl_napi_fill_one(rsp, napi, info);
222 } else {
223 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
224 err = -ENOENT;
225 }
226
227 rtnl_unlock();
228
229 if (err)
230 goto err_free_msg;
231
232 return genlmsg_reply(rsp, info);
233
234 err_free_msg:
235 nlmsg_free(rsp);
236 return err;
237 }
238
239 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)240 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
241 const struct genl_info *info,
242 struct netdev_nl_dump_ctx *ctx)
243 {
244 struct napi_struct *napi;
245 int err = 0;
246
247 if (!(netdev->flags & IFF_UP))
248 return err;
249
250 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
251 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
252 continue;
253
254 err = netdev_nl_napi_fill_one(rsp, napi, info);
255 if (err)
256 return err;
257 ctx->napi_id = napi->napi_id;
258 }
259 return err;
260 }
261
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)262 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
263 {
264 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
265 const struct genl_info *info = genl_info_dump(cb);
266 struct net *net = sock_net(skb->sk);
267 struct net_device *netdev;
268 u32 ifindex = 0;
269 int err = 0;
270
271 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
272 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
273
274 rtnl_lock();
275 if (ifindex) {
276 netdev = __dev_get_by_index(net, ifindex);
277 if (netdev)
278 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
279 else
280 err = -ENODEV;
281 } else {
282 for_each_netdev_dump(net, netdev, ctx->ifindex) {
283 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
284 if (err < 0)
285 break;
286 ctx->napi_id = 0;
287 }
288 }
289 rtnl_unlock();
290
291 return err;
292 }
293
294 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)295 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
296 u32 q_idx, u32 q_type, const struct genl_info *info)
297 {
298 struct net_devmem_dmabuf_binding *binding;
299 struct netdev_rx_queue *rxq;
300 struct netdev_queue *txq;
301 void *hdr;
302
303 hdr = genlmsg_iput(rsp, info);
304 if (!hdr)
305 return -EMSGSIZE;
306
307 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
308 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
309 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
310 goto nla_put_failure;
311
312 switch (q_type) {
313 case NETDEV_QUEUE_TYPE_RX:
314 rxq = __netif_get_rx_queue(netdev, q_idx);
315 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
316 rxq->napi->napi_id))
317 goto nla_put_failure;
318
319 binding = rxq->mp_params.mp_priv;
320 if (binding &&
321 nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id))
322 goto nla_put_failure;
323
324 break;
325 case NETDEV_QUEUE_TYPE_TX:
326 txq = netdev_get_tx_queue(netdev, q_idx);
327 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID,
328 txq->napi->napi_id))
329 goto nla_put_failure;
330 }
331
332 genlmsg_end(rsp, hdr);
333
334 return 0;
335
336 nla_put_failure:
337 genlmsg_cancel(rsp, hdr);
338 return -EMSGSIZE;
339 }
340
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)341 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
342 u32 q_type)
343 {
344 switch (q_type) {
345 case NETDEV_QUEUE_TYPE_RX:
346 if (q_id >= netdev->real_num_rx_queues)
347 return -EINVAL;
348 return 0;
349 case NETDEV_QUEUE_TYPE_TX:
350 if (q_id >= netdev->real_num_tx_queues)
351 return -EINVAL;
352 }
353 return 0;
354 }
355
356 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)357 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
358 u32 q_type, const struct genl_info *info)
359 {
360 int err = 0;
361
362 if (!(netdev->flags & IFF_UP))
363 return err;
364
365 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
366 if (err)
367 return err;
368
369 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
370 }
371
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)372 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
373 {
374 u32 q_id, q_type, ifindex;
375 struct net_device *netdev;
376 struct sk_buff *rsp;
377 int err;
378
379 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
380 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
381 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
382 return -EINVAL;
383
384 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
385 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
386 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
387
388 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
389 if (!rsp)
390 return -ENOMEM;
391
392 rtnl_lock();
393
394 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
395 if (netdev)
396 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
397 else
398 err = -ENODEV;
399
400 rtnl_unlock();
401
402 if (err)
403 goto err_free_msg;
404
405 return genlmsg_reply(rsp, info);
406
407 err_free_msg:
408 nlmsg_free(rsp);
409 return err;
410 }
411
412 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)413 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
414 const struct genl_info *info,
415 struct netdev_nl_dump_ctx *ctx)
416 {
417 int err = 0;
418 int i;
419
420 if (!(netdev->flags & IFF_UP))
421 return err;
422
423 for (i = ctx->rxq_idx; i < netdev->real_num_rx_queues;) {
424 err = netdev_nl_queue_fill_one(rsp, netdev, i,
425 NETDEV_QUEUE_TYPE_RX, info);
426 if (err)
427 return err;
428 ctx->rxq_idx = i++;
429 }
430 for (i = ctx->txq_idx; i < netdev->real_num_tx_queues;) {
431 err = netdev_nl_queue_fill_one(rsp, netdev, i,
432 NETDEV_QUEUE_TYPE_TX, info);
433 if (err)
434 return err;
435 ctx->txq_idx = i++;
436 }
437
438 return err;
439 }
440
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)441 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
442 {
443 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
444 const struct genl_info *info = genl_info_dump(cb);
445 struct net *net = sock_net(skb->sk);
446 struct net_device *netdev;
447 u32 ifindex = 0;
448 int err = 0;
449
450 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
451 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
452
453 rtnl_lock();
454 if (ifindex) {
455 netdev = __dev_get_by_index(net, ifindex);
456 if (netdev)
457 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
458 else
459 err = -ENODEV;
460 } else {
461 for_each_netdev_dump(net, netdev, ctx->ifindex) {
462 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
463 if (err < 0)
464 break;
465 ctx->rxq_idx = 0;
466 ctx->txq_idx = 0;
467 }
468 }
469 rtnl_unlock();
470
471 return err;
472 }
473
474 #define NETDEV_STAT_NOT_SET (~0ULL)
475
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)476 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
477 {
478 const u64 *add = _add;
479 u64 *sum = _sum;
480
481 while (size) {
482 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
483 *sum += *add;
484 sum++;
485 add++;
486 size -= 8;
487 }
488 }
489
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)490 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
491 {
492 if (value == NETDEV_STAT_NOT_SET)
493 return 0;
494 return nla_put_uint(rsp, attr_id, value);
495 }
496
497 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)498 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
499 {
500 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
501 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
502 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
503 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
504 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
505 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
506 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
507 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
508 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
509 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
510 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
511 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
512 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
513 return -EMSGSIZE;
514 return 0;
515 }
516
517 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)518 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
519 {
520 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
521 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
522 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
523 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
524 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
525 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
526 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
527 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
528 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
529 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
530 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
531 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
532 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
533 return -EMSGSIZE;
534 return 0;
535 }
536
537 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)538 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
539 u32 q_type, int i, const struct genl_info *info)
540 {
541 const struct netdev_stat_ops *ops = netdev->stat_ops;
542 struct netdev_queue_stats_rx rx;
543 struct netdev_queue_stats_tx tx;
544 void *hdr;
545
546 hdr = genlmsg_iput(rsp, info);
547 if (!hdr)
548 return -EMSGSIZE;
549 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
550 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
551 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
552 goto nla_put_failure;
553
554 switch (q_type) {
555 case NETDEV_QUEUE_TYPE_RX:
556 memset(&rx, 0xff, sizeof(rx));
557 ops->get_queue_stats_rx(netdev, i, &rx);
558 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
559 goto nla_cancel;
560 if (netdev_nl_stats_write_rx(rsp, &rx))
561 goto nla_put_failure;
562 break;
563 case NETDEV_QUEUE_TYPE_TX:
564 memset(&tx, 0xff, sizeof(tx));
565 ops->get_queue_stats_tx(netdev, i, &tx);
566 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
567 goto nla_cancel;
568 if (netdev_nl_stats_write_tx(rsp, &tx))
569 goto nla_put_failure;
570 break;
571 }
572
573 genlmsg_end(rsp, hdr);
574 return 0;
575
576 nla_cancel:
577 genlmsg_cancel(rsp, hdr);
578 return 0;
579 nla_put_failure:
580 genlmsg_cancel(rsp, hdr);
581 return -EMSGSIZE;
582 }
583
584 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)585 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
586 const struct genl_info *info,
587 struct netdev_nl_dump_ctx *ctx)
588 {
589 const struct netdev_stat_ops *ops = netdev->stat_ops;
590 int i, err;
591
592 if (!(netdev->flags & IFF_UP))
593 return 0;
594
595 i = ctx->rxq_idx;
596 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
597 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
598 i, info);
599 if (err)
600 return err;
601 ctx->rxq_idx = i++;
602 }
603 i = ctx->txq_idx;
604 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
605 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
606 i, info);
607 if (err)
608 return err;
609 ctx->txq_idx = i++;
610 }
611
612 ctx->rxq_idx = 0;
613 ctx->txq_idx = 0;
614 return 0;
615 }
616
617 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)618 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
619 const struct genl_info *info)
620 {
621 struct netdev_queue_stats_rx rx_sum, rx;
622 struct netdev_queue_stats_tx tx_sum, tx;
623 const struct netdev_stat_ops *ops;
624 void *hdr;
625 int i;
626
627 ops = netdev->stat_ops;
628 /* Netdev can't guarantee any complete counters */
629 if (!ops->get_base_stats)
630 return 0;
631
632 memset(&rx_sum, 0xff, sizeof(rx_sum));
633 memset(&tx_sum, 0xff, sizeof(tx_sum));
634
635 ops->get_base_stats(netdev, &rx_sum, &tx_sum);
636
637 /* The op was there, but nothing reported, don't bother */
638 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
639 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
640 return 0;
641
642 hdr = genlmsg_iput(rsp, info);
643 if (!hdr)
644 return -EMSGSIZE;
645 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
646 goto nla_put_failure;
647
648 for (i = 0; i < netdev->real_num_rx_queues; i++) {
649 memset(&rx, 0xff, sizeof(rx));
650 if (ops->get_queue_stats_rx)
651 ops->get_queue_stats_rx(netdev, i, &rx);
652 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx));
653 }
654 for (i = 0; i < netdev->real_num_tx_queues; i++) {
655 memset(&tx, 0xff, sizeof(tx));
656 if (ops->get_queue_stats_tx)
657 ops->get_queue_stats_tx(netdev, i, &tx);
658 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx));
659 }
660
661 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
662 netdev_nl_stats_write_tx(rsp, &tx_sum))
663 goto nla_put_failure;
664
665 genlmsg_end(rsp, hdr);
666 return 0;
667
668 nla_put_failure:
669 genlmsg_cancel(rsp, hdr);
670 return -EMSGSIZE;
671 }
672
673 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)674 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
675 struct sk_buff *skb, const struct genl_info *info,
676 struct netdev_nl_dump_ctx *ctx)
677 {
678 if (!netdev->stat_ops)
679 return 0;
680
681 switch (scope) {
682 case 0:
683 return netdev_nl_stats_by_netdev(netdev, skb, info);
684 case NETDEV_QSTATS_SCOPE_QUEUE:
685 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
686 }
687
688 return -EINVAL; /* Should not happen, per netlink policy */
689 }
690
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)691 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
692 struct netlink_callback *cb)
693 {
694 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
695 const struct genl_info *info = genl_info_dump(cb);
696 struct net *net = sock_net(skb->sk);
697 struct net_device *netdev;
698 unsigned int ifindex;
699 unsigned int scope;
700 int err = 0;
701
702 scope = 0;
703 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
704 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
705
706 ifindex = 0;
707 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
708 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
709
710 rtnl_lock();
711 if (ifindex) {
712 netdev = __dev_get_by_index(net, ifindex);
713 if (netdev && netdev->stat_ops) {
714 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
715 info, ctx);
716 } else {
717 NL_SET_BAD_ATTR(info->extack,
718 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
719 err = netdev ? -EOPNOTSUPP : -ENODEV;
720 }
721 } else {
722 for_each_netdev_dump(net, netdev, ctx->ifindex) {
723 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
724 info, ctx);
725 if (err < 0)
726 break;
727 }
728 }
729 rtnl_unlock();
730
731 return err;
732 }
733
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)734 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
735 {
736 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
737 struct net_devmem_dmabuf_binding *binding;
738 struct list_head *sock_binding_list;
739 u32 ifindex, dmabuf_fd, rxq_idx;
740 struct net_device *netdev;
741 struct sk_buff *rsp;
742 struct nlattr *attr;
743 int rem, err = 0;
744 void *hdr;
745
746 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
747 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
748 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
749 return -EINVAL;
750
751 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
752 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
753
754 sock_binding_list = genl_sk_priv_get(&netdev_nl_family,
755 NETLINK_CB(skb).sk);
756 if (IS_ERR(sock_binding_list))
757 return PTR_ERR(sock_binding_list);
758
759 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
760 if (!rsp)
761 return -ENOMEM;
762
763 hdr = genlmsg_iput(rsp, info);
764 if (!hdr) {
765 err = -EMSGSIZE;
766 goto err_genlmsg_free;
767 }
768
769 rtnl_lock();
770
771 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
772 if (!netdev || !netif_device_present(netdev)) {
773 err = -ENODEV;
774 goto err_unlock;
775 }
776
777 if (dev_xdp_prog_count(netdev)) {
778 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached");
779 err = -EEXIST;
780 goto err_unlock;
781 }
782
783 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
784 if (IS_ERR(binding)) {
785 err = PTR_ERR(binding);
786 goto err_unlock;
787 }
788
789 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
790 genlmsg_data(info->genlhdr),
791 genlmsg_len(info->genlhdr), rem) {
792 err = nla_parse_nested(
793 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
794 netdev_queue_id_nl_policy, info->extack);
795 if (err < 0)
796 goto err_unbind;
797
798 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
799 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
800 err = -EINVAL;
801 goto err_unbind;
802 }
803
804 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
805 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
806 err = -EINVAL;
807 goto err_unbind;
808 }
809
810 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
811
812 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
813 info->extack);
814 if (err)
815 goto err_unbind;
816 }
817
818 list_add(&binding->list, sock_binding_list);
819
820 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
821 genlmsg_end(rsp, hdr);
822
823 err = genlmsg_reply(rsp, info);
824 if (err)
825 goto err_unbind;
826
827 rtnl_unlock();
828
829 return 0;
830
831 err_unbind:
832 net_devmem_unbind_dmabuf(binding);
833 err_unlock:
834 rtnl_unlock();
835 err_genlmsg_free:
836 nlmsg_free(rsp);
837 return err;
838 }
839
netdev_nl_sock_priv_init(struct list_head * priv)840 void netdev_nl_sock_priv_init(struct list_head *priv)
841 {
842 INIT_LIST_HEAD(priv);
843 }
844
netdev_nl_sock_priv_destroy(struct list_head * priv)845 void netdev_nl_sock_priv_destroy(struct list_head *priv)
846 {
847 struct net_devmem_dmabuf_binding *binding;
848 struct net_devmem_dmabuf_binding *temp;
849
850 list_for_each_entry_safe(binding, temp, priv, list) {
851 rtnl_lock();
852 net_devmem_unbind_dmabuf(binding);
853 rtnl_unlock();
854 }
855 }
856
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)857 static int netdev_genl_netdevice_event(struct notifier_block *nb,
858 unsigned long event, void *ptr)
859 {
860 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
861
862 switch (event) {
863 case NETDEV_REGISTER:
864 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
865 break;
866 case NETDEV_UNREGISTER:
867 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
868 break;
869 case NETDEV_XDP_FEAT_CHANGE:
870 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
871 break;
872 }
873
874 return NOTIFY_OK;
875 }
876
877 static struct notifier_block netdev_genl_nb = {
878 .notifier_call = netdev_genl_netdevice_event,
879 };
880
netdev_genl_init(void)881 static int __init netdev_genl_init(void)
882 {
883 int err;
884
885 err = register_netdevice_notifier(&netdev_genl_nb);
886 if (err)
887 return err;
888
889 err = genl_register_family(&netdev_nl_family);
890 if (err)
891 goto err_unreg_ntf;
892
893 return 0;
894
895 err_unreg_ntf:
896 unregister_netdevice_notifier(&netdev_genl_nb);
897 return err;
898 }
899
900 subsys_initcall(netdev_genl_init);
901