1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/rtnetlink.h> 6 #include <net/busy_poll.h> 7 #include <net/net_namespace.h> 8 #include <net/netdev_queues.h> 9 #include <net/netdev_rx_queue.h> 10 #include <net/sock.h> 11 #include <net/xdp.h> 12 #include <net/xdp_sock.h> 13 14 #include "dev.h" 15 #include "devmem.h" 16 #include "netdev-genl-gen.h" 17 18 struct netdev_nl_dump_ctx { 19 unsigned long ifindex; 20 unsigned int rxq_idx; 21 unsigned int txq_idx; 22 unsigned int napi_id; 23 }; 24 25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 26 { 27 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 28 29 return (struct netdev_nl_dump_ctx *)cb->ctx; 30 } 31 32 static int 33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 34 const struct genl_info *info) 35 { 36 u64 xsk_features = 0; 37 u64 xdp_rx_meta = 0; 38 void *hdr; 39 40 hdr = genlmsg_iput(rsp, info); 41 if (!hdr) 42 return -EMSGSIZE; 43 44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 45 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 46 xdp_rx_meta |= flag; 47 XDP_METADATA_KFUNC_xxx 48 #undef XDP_METADATA_KFUNC 49 50 if (netdev->xsk_tx_metadata_ops) { 51 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 52 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 53 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 54 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 55 } 56 57 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 58 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 59 netdev->xdp_features, NETDEV_A_DEV_PAD) || 60 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 61 xdp_rx_meta, NETDEV_A_DEV_PAD) || 62 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 63 xsk_features, NETDEV_A_DEV_PAD)) 64 goto err_cancel_msg; 65 66 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 67 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 68 netdev->xdp_zc_max_segs)) 69 goto err_cancel_msg; 70 } 71 72 genlmsg_end(rsp, hdr); 73 74 return 0; 75 76 err_cancel_msg: 77 genlmsg_cancel(rsp, hdr); 78 return -EMSGSIZE; 79 } 80 81 static void 82 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 83 { 84 struct genl_info info; 85 struct sk_buff *ntf; 86 87 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 88 NETDEV_NLGRP_MGMT)) 89 return; 90 91 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 92 93 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 94 if (!ntf) 95 return; 96 97 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 98 nlmsg_free(ntf); 99 return; 100 } 101 102 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 103 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 104 } 105 106 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 107 { 108 struct net_device *netdev; 109 struct sk_buff *rsp; 110 u32 ifindex; 111 int err; 112 113 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 114 return -EINVAL; 115 116 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 117 118 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 119 if (!rsp) 120 return -ENOMEM; 121 122 rtnl_lock(); 123 124 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 125 if (netdev) 126 err = netdev_nl_dev_fill(netdev, rsp, info); 127 else 128 err = -ENODEV; 129 130 rtnl_unlock(); 131 132 if (err) 133 goto err_free_msg; 134 135 return genlmsg_reply(rsp, info); 136 137 err_free_msg: 138 nlmsg_free(rsp); 139 return err; 140 } 141 142 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 143 { 144 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 145 struct net *net = sock_net(skb->sk); 146 struct net_device *netdev; 147 int err = 0; 148 149 rtnl_lock(); 150 for_each_netdev_dump(net, netdev, ctx->ifindex) { 151 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 152 if (err < 0) 153 break; 154 } 155 rtnl_unlock(); 156 157 return err; 158 } 159 160 static int 161 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 162 const struct genl_info *info) 163 { 164 unsigned long irq_suspend_timeout; 165 unsigned long gro_flush_timeout; 166 u32 napi_defer_hard_irqs; 167 void *hdr; 168 pid_t pid; 169 170 if (WARN_ON_ONCE(!napi->dev)) 171 return -EINVAL; 172 if (!(napi->dev->flags & IFF_UP)) 173 return 0; 174 175 hdr = genlmsg_iput(rsp, info); 176 if (!hdr) 177 return -EMSGSIZE; 178 179 if (napi->napi_id >= MIN_NAPI_ID && 180 nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 181 goto nla_put_failure; 182 183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 184 goto nla_put_failure; 185 186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 187 goto nla_put_failure; 188 189 if (napi->thread) { 190 pid = task_pid_nr(napi->thread); 191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 192 goto nla_put_failure; 193 } 194 195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 197 napi_defer_hard_irqs)) 198 goto nla_put_failure; 199 200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 202 irq_suspend_timeout)) 203 goto nla_put_failure; 204 205 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 207 gro_flush_timeout)) 208 goto nla_put_failure; 209 210 genlmsg_end(rsp, hdr); 211 212 return 0; 213 214 nla_put_failure: 215 genlmsg_cancel(rsp, hdr); 216 return -EMSGSIZE; 217 } 218 219 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 220 { 221 struct napi_struct *napi; 222 struct sk_buff *rsp; 223 u32 napi_id; 224 int err; 225 226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 227 return -EINVAL; 228 229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 230 231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 232 if (!rsp) 233 return -ENOMEM; 234 235 rtnl_lock(); 236 237 napi = napi_by_id(napi_id); 238 if (napi) { 239 err = netdev_nl_napi_fill_one(rsp, napi, info); 240 } else { 241 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 242 err = -ENOENT; 243 } 244 245 rtnl_unlock(); 246 247 if (err) 248 goto err_free_msg; 249 250 return genlmsg_reply(rsp, info); 251 252 err_free_msg: 253 nlmsg_free(rsp); 254 return err; 255 } 256 257 static int 258 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 259 const struct genl_info *info, 260 struct netdev_nl_dump_ctx *ctx) 261 { 262 struct napi_struct *napi; 263 int err = 0; 264 265 if (!(netdev->flags & IFF_UP)) 266 return err; 267 268 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 269 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 270 continue; 271 272 err = netdev_nl_napi_fill_one(rsp, napi, info); 273 if (err) 274 return err; 275 ctx->napi_id = napi->napi_id; 276 } 277 return err; 278 } 279 280 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 281 { 282 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 283 const struct genl_info *info = genl_info_dump(cb); 284 struct net *net = sock_net(skb->sk); 285 struct net_device *netdev; 286 u32 ifindex = 0; 287 int err = 0; 288 289 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 290 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 291 292 rtnl_lock(); 293 if (ifindex) { 294 netdev = __dev_get_by_index(net, ifindex); 295 if (netdev) 296 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 297 else 298 err = -ENODEV; 299 } else { 300 for_each_netdev_dump(net, netdev, ctx->ifindex) { 301 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 302 if (err < 0) 303 break; 304 ctx->napi_id = 0; 305 } 306 } 307 rtnl_unlock(); 308 309 return err; 310 } 311 312 static int 313 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 314 { 315 u64 irq_suspend_timeout = 0; 316 u64 gro_flush_timeout = 0; 317 u32 defer = 0; 318 319 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 320 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 321 napi_set_defer_hard_irqs(napi, defer); 322 } 323 324 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 325 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 326 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 327 } 328 329 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 330 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 331 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 332 } 333 334 return 0; 335 } 336 337 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 338 { 339 struct napi_struct *napi; 340 unsigned int napi_id; 341 int err; 342 343 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 344 return -EINVAL; 345 346 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 347 348 rtnl_lock(); 349 rcu_read_lock(); 350 351 napi = napi_by_id(napi_id); 352 if (napi) { 353 err = netdev_nl_napi_set_config(napi, info); 354 } else { 355 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 356 err = -ENOENT; 357 } 358 359 rcu_read_unlock(); 360 rtnl_unlock(); 361 362 return err; 363 } 364 365 static int 366 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 367 u32 q_idx, u32 q_type, const struct genl_info *info) 368 { 369 struct net_devmem_dmabuf_binding *binding; 370 struct netdev_rx_queue *rxq; 371 struct netdev_queue *txq; 372 void *hdr; 373 374 hdr = genlmsg_iput(rsp, info); 375 if (!hdr) 376 return -EMSGSIZE; 377 378 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 379 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 380 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 381 goto nla_put_failure; 382 383 switch (q_type) { 384 case NETDEV_QUEUE_TYPE_RX: 385 rxq = __netif_get_rx_queue(netdev, q_idx); 386 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID, 387 rxq->napi->napi_id)) 388 goto nla_put_failure; 389 390 binding = rxq->mp_params.mp_priv; 391 if (binding && 392 nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id)) 393 goto nla_put_failure; 394 395 break; 396 case NETDEV_QUEUE_TYPE_TX: 397 txq = netdev_get_tx_queue(netdev, q_idx); 398 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID, 399 txq->napi->napi_id)) 400 goto nla_put_failure; 401 } 402 403 genlmsg_end(rsp, hdr); 404 405 return 0; 406 407 nla_put_failure: 408 genlmsg_cancel(rsp, hdr); 409 return -EMSGSIZE; 410 } 411 412 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 413 u32 q_type) 414 { 415 switch (q_type) { 416 case NETDEV_QUEUE_TYPE_RX: 417 if (q_id >= netdev->real_num_rx_queues) 418 return -EINVAL; 419 return 0; 420 case NETDEV_QUEUE_TYPE_TX: 421 if (q_id >= netdev->real_num_tx_queues) 422 return -EINVAL; 423 } 424 return 0; 425 } 426 427 static int 428 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 429 u32 q_type, const struct genl_info *info) 430 { 431 int err = 0; 432 433 if (!(netdev->flags & IFF_UP)) 434 return err; 435 436 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 437 if (err) 438 return err; 439 440 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 441 } 442 443 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 444 { 445 u32 q_id, q_type, ifindex; 446 struct net_device *netdev; 447 struct sk_buff *rsp; 448 int err; 449 450 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 451 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 452 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 453 return -EINVAL; 454 455 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 456 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 457 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 458 459 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 460 if (!rsp) 461 return -ENOMEM; 462 463 rtnl_lock(); 464 465 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 466 if (netdev) 467 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 468 else 469 err = -ENODEV; 470 471 rtnl_unlock(); 472 473 if (err) 474 goto err_free_msg; 475 476 return genlmsg_reply(rsp, info); 477 478 err_free_msg: 479 nlmsg_free(rsp); 480 return err; 481 } 482 483 static int 484 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 485 const struct genl_info *info, 486 struct netdev_nl_dump_ctx *ctx) 487 { 488 int err = 0; 489 int i; 490 491 if (!(netdev->flags & IFF_UP)) 492 return err; 493 494 for (i = ctx->rxq_idx; i < netdev->real_num_rx_queues;) { 495 err = netdev_nl_queue_fill_one(rsp, netdev, i, 496 NETDEV_QUEUE_TYPE_RX, info); 497 if (err) 498 return err; 499 ctx->rxq_idx = i++; 500 } 501 for (i = ctx->txq_idx; i < netdev->real_num_tx_queues;) { 502 err = netdev_nl_queue_fill_one(rsp, netdev, i, 503 NETDEV_QUEUE_TYPE_TX, info); 504 if (err) 505 return err; 506 ctx->txq_idx = i++; 507 } 508 509 return err; 510 } 511 512 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 513 { 514 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 515 const struct genl_info *info = genl_info_dump(cb); 516 struct net *net = sock_net(skb->sk); 517 struct net_device *netdev; 518 u32 ifindex = 0; 519 int err = 0; 520 521 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 522 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 523 524 rtnl_lock(); 525 if (ifindex) { 526 netdev = __dev_get_by_index(net, ifindex); 527 if (netdev) 528 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 529 else 530 err = -ENODEV; 531 } else { 532 for_each_netdev_dump(net, netdev, ctx->ifindex) { 533 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 534 if (err < 0) 535 break; 536 ctx->rxq_idx = 0; 537 ctx->txq_idx = 0; 538 } 539 } 540 rtnl_unlock(); 541 542 return err; 543 } 544 545 #define NETDEV_STAT_NOT_SET (~0ULL) 546 547 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 548 { 549 const u64 *add = _add; 550 u64 *sum = _sum; 551 552 while (size) { 553 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 554 *sum += *add; 555 sum++; 556 add++; 557 size -= 8; 558 } 559 } 560 561 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 562 { 563 if (value == NETDEV_STAT_NOT_SET) 564 return 0; 565 return nla_put_uint(rsp, attr_id, value); 566 } 567 568 static int 569 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 570 { 571 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 572 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 573 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 574 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 575 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 576 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 577 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 578 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 579 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 580 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 581 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 582 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 583 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 584 return -EMSGSIZE; 585 return 0; 586 } 587 588 static int 589 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 590 { 591 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 592 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 593 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 594 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 595 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 596 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 597 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 598 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 599 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 600 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 601 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 602 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 603 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 604 return -EMSGSIZE; 605 return 0; 606 } 607 608 static int 609 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 610 u32 q_type, int i, const struct genl_info *info) 611 { 612 const struct netdev_stat_ops *ops = netdev->stat_ops; 613 struct netdev_queue_stats_rx rx; 614 struct netdev_queue_stats_tx tx; 615 void *hdr; 616 617 hdr = genlmsg_iput(rsp, info); 618 if (!hdr) 619 return -EMSGSIZE; 620 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 621 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 622 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 623 goto nla_put_failure; 624 625 switch (q_type) { 626 case NETDEV_QUEUE_TYPE_RX: 627 memset(&rx, 0xff, sizeof(rx)); 628 ops->get_queue_stats_rx(netdev, i, &rx); 629 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 630 goto nla_cancel; 631 if (netdev_nl_stats_write_rx(rsp, &rx)) 632 goto nla_put_failure; 633 break; 634 case NETDEV_QUEUE_TYPE_TX: 635 memset(&tx, 0xff, sizeof(tx)); 636 ops->get_queue_stats_tx(netdev, i, &tx); 637 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 638 goto nla_cancel; 639 if (netdev_nl_stats_write_tx(rsp, &tx)) 640 goto nla_put_failure; 641 break; 642 } 643 644 genlmsg_end(rsp, hdr); 645 return 0; 646 647 nla_cancel: 648 genlmsg_cancel(rsp, hdr); 649 return 0; 650 nla_put_failure: 651 genlmsg_cancel(rsp, hdr); 652 return -EMSGSIZE; 653 } 654 655 static int 656 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 657 const struct genl_info *info, 658 struct netdev_nl_dump_ctx *ctx) 659 { 660 const struct netdev_stat_ops *ops = netdev->stat_ops; 661 int i, err; 662 663 if (!(netdev->flags & IFF_UP)) 664 return 0; 665 666 i = ctx->rxq_idx; 667 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 668 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 669 i, info); 670 if (err) 671 return err; 672 ctx->rxq_idx = i++; 673 } 674 i = ctx->txq_idx; 675 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 676 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 677 i, info); 678 if (err) 679 return err; 680 ctx->txq_idx = i++; 681 } 682 683 ctx->rxq_idx = 0; 684 ctx->txq_idx = 0; 685 return 0; 686 } 687 688 static int 689 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 690 const struct genl_info *info) 691 { 692 struct netdev_queue_stats_rx rx_sum, rx; 693 struct netdev_queue_stats_tx tx_sum, tx; 694 const struct netdev_stat_ops *ops; 695 void *hdr; 696 int i; 697 698 ops = netdev->stat_ops; 699 /* Netdev can't guarantee any complete counters */ 700 if (!ops->get_base_stats) 701 return 0; 702 703 memset(&rx_sum, 0xff, sizeof(rx_sum)); 704 memset(&tx_sum, 0xff, sizeof(tx_sum)); 705 706 ops->get_base_stats(netdev, &rx_sum, &tx_sum); 707 708 /* The op was there, but nothing reported, don't bother */ 709 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 710 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 711 return 0; 712 713 hdr = genlmsg_iput(rsp, info); 714 if (!hdr) 715 return -EMSGSIZE; 716 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 717 goto nla_put_failure; 718 719 for (i = 0; i < netdev->real_num_rx_queues; i++) { 720 memset(&rx, 0xff, sizeof(rx)); 721 if (ops->get_queue_stats_rx) 722 ops->get_queue_stats_rx(netdev, i, &rx); 723 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx)); 724 } 725 for (i = 0; i < netdev->real_num_tx_queues; i++) { 726 memset(&tx, 0xff, sizeof(tx)); 727 if (ops->get_queue_stats_tx) 728 ops->get_queue_stats_tx(netdev, i, &tx); 729 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx)); 730 } 731 732 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 733 netdev_nl_stats_write_tx(rsp, &tx_sum)) 734 goto nla_put_failure; 735 736 genlmsg_end(rsp, hdr); 737 return 0; 738 739 nla_put_failure: 740 genlmsg_cancel(rsp, hdr); 741 return -EMSGSIZE; 742 } 743 744 static int 745 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 746 struct sk_buff *skb, const struct genl_info *info, 747 struct netdev_nl_dump_ctx *ctx) 748 { 749 if (!netdev->stat_ops) 750 return 0; 751 752 switch (scope) { 753 case 0: 754 return netdev_nl_stats_by_netdev(netdev, skb, info); 755 case NETDEV_QSTATS_SCOPE_QUEUE: 756 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 757 } 758 759 return -EINVAL; /* Should not happen, per netlink policy */ 760 } 761 762 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 763 struct netlink_callback *cb) 764 { 765 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 766 const struct genl_info *info = genl_info_dump(cb); 767 struct net *net = sock_net(skb->sk); 768 struct net_device *netdev; 769 unsigned int ifindex; 770 unsigned int scope; 771 int err = 0; 772 773 scope = 0; 774 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 775 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 776 777 ifindex = 0; 778 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 779 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 780 781 rtnl_lock(); 782 if (ifindex) { 783 netdev = __dev_get_by_index(net, ifindex); 784 if (netdev && netdev->stat_ops) { 785 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 786 info, ctx); 787 } else { 788 NL_SET_BAD_ATTR(info->extack, 789 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 790 err = netdev ? -EOPNOTSUPP : -ENODEV; 791 } 792 } else { 793 for_each_netdev_dump(net, netdev, ctx->ifindex) { 794 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 795 info, ctx); 796 if (err < 0) 797 break; 798 } 799 } 800 rtnl_unlock(); 801 802 return err; 803 } 804 805 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 806 { 807 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 808 struct net_devmem_dmabuf_binding *binding; 809 struct list_head *sock_binding_list; 810 u32 ifindex, dmabuf_fd, rxq_idx; 811 struct net_device *netdev; 812 struct sk_buff *rsp; 813 struct nlattr *attr; 814 int rem, err = 0; 815 void *hdr; 816 817 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 818 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 819 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 820 return -EINVAL; 821 822 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 823 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 824 825 sock_binding_list = genl_sk_priv_get(&netdev_nl_family, 826 NETLINK_CB(skb).sk); 827 if (IS_ERR(sock_binding_list)) 828 return PTR_ERR(sock_binding_list); 829 830 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 831 if (!rsp) 832 return -ENOMEM; 833 834 hdr = genlmsg_iput(rsp, info); 835 if (!hdr) { 836 err = -EMSGSIZE; 837 goto err_genlmsg_free; 838 } 839 840 rtnl_lock(); 841 842 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 843 if (!netdev || !netif_device_present(netdev)) { 844 err = -ENODEV; 845 goto err_unlock; 846 } 847 848 if (dev_xdp_prog_count(netdev)) { 849 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached"); 850 err = -EEXIST; 851 goto err_unlock; 852 } 853 854 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack); 855 if (IS_ERR(binding)) { 856 err = PTR_ERR(binding); 857 goto err_unlock; 858 } 859 860 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 861 genlmsg_data(info->genlhdr), 862 genlmsg_len(info->genlhdr), rem) { 863 err = nla_parse_nested( 864 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr, 865 netdev_queue_id_nl_policy, info->extack); 866 if (err < 0) 867 goto err_unbind; 868 869 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 870 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) { 871 err = -EINVAL; 872 goto err_unbind; 873 } 874 875 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 876 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 877 err = -EINVAL; 878 goto err_unbind; 879 } 880 881 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 882 883 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 884 info->extack); 885 if (err) 886 goto err_unbind; 887 } 888 889 list_add(&binding->list, sock_binding_list); 890 891 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 892 genlmsg_end(rsp, hdr); 893 894 err = genlmsg_reply(rsp, info); 895 if (err) 896 goto err_unbind; 897 898 rtnl_unlock(); 899 900 return 0; 901 902 err_unbind: 903 net_devmem_unbind_dmabuf(binding); 904 err_unlock: 905 rtnl_unlock(); 906 err_genlmsg_free: 907 nlmsg_free(rsp); 908 return err; 909 } 910 911 void netdev_nl_sock_priv_init(struct list_head *priv) 912 { 913 INIT_LIST_HEAD(priv); 914 } 915 916 void netdev_nl_sock_priv_destroy(struct list_head *priv) 917 { 918 struct net_devmem_dmabuf_binding *binding; 919 struct net_devmem_dmabuf_binding *temp; 920 921 list_for_each_entry_safe(binding, temp, priv, list) { 922 rtnl_lock(); 923 net_devmem_unbind_dmabuf(binding); 924 rtnl_unlock(); 925 } 926 } 927 928 static int netdev_genl_netdevice_event(struct notifier_block *nb, 929 unsigned long event, void *ptr) 930 { 931 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 932 933 switch (event) { 934 case NETDEV_REGISTER: 935 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 936 break; 937 case NETDEV_UNREGISTER: 938 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 939 break; 940 case NETDEV_XDP_FEAT_CHANGE: 941 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 942 break; 943 } 944 945 return NOTIFY_OK; 946 } 947 948 static struct notifier_block netdev_genl_nb = { 949 .notifier_call = netdev_genl_netdevice_event, 950 }; 951 952 static int __init netdev_genl_init(void) 953 { 954 int err; 955 956 err = register_netdevice_notifier(&netdev_genl_nb); 957 if (err) 958 return err; 959 960 err = genl_register_family(&netdev_nl_family); 961 if (err) 962 goto err_unreg_ntf; 963 964 return 0; 965 966 err_unreg_ntf: 967 unregister_netdevice_notifier(&netdev_genl_nb); 968 return err; 969 } 970 971 subsys_initcall(netdev_genl_init); 972