1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/rtnetlink.h> 6 #include <net/busy_poll.h> 7 #include <net/net_namespace.h> 8 #include <net/netdev_queues.h> 9 #include <net/netdev_rx_queue.h> 10 #include <net/sock.h> 11 #include <net/xdp.h> 12 #include <net/xdp_sock.h> 13 14 #include "dev.h" 15 #include "devmem.h" 16 #include "netdev-genl-gen.h" 17 18 struct netdev_nl_dump_ctx { 19 unsigned long ifindex; 20 unsigned int rxq_idx; 21 unsigned int txq_idx; 22 unsigned int napi_id; 23 }; 24 25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 26 { 27 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 28 29 return (struct netdev_nl_dump_ctx *)cb->ctx; 30 } 31 32 static int 33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 34 const struct genl_info *info) 35 { 36 u64 xsk_features = 0; 37 u64 xdp_rx_meta = 0; 38 void *hdr; 39 40 hdr = genlmsg_iput(rsp, info); 41 if (!hdr) 42 return -EMSGSIZE; 43 44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 45 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 46 xdp_rx_meta |= flag; 47 XDP_METADATA_KFUNC_xxx 48 #undef XDP_METADATA_KFUNC 49 50 if (netdev->xsk_tx_metadata_ops) { 51 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 52 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 53 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 54 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 55 } 56 57 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 58 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 59 netdev->xdp_features, NETDEV_A_DEV_PAD) || 60 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 61 xdp_rx_meta, NETDEV_A_DEV_PAD) || 62 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 63 xsk_features, NETDEV_A_DEV_PAD)) 64 goto err_cancel_msg; 65 66 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 67 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 68 netdev->xdp_zc_max_segs)) 69 goto err_cancel_msg; 70 } 71 72 genlmsg_end(rsp, hdr); 73 74 return 0; 75 76 err_cancel_msg: 77 genlmsg_cancel(rsp, hdr); 78 return -EMSGSIZE; 79 } 80 81 static void 82 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 83 { 84 struct genl_info info; 85 struct sk_buff *ntf; 86 87 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 88 NETDEV_NLGRP_MGMT)) 89 return; 90 91 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 92 93 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 94 if (!ntf) 95 return; 96 97 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 98 nlmsg_free(ntf); 99 return; 100 } 101 102 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 103 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 104 } 105 106 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 107 { 108 struct net_device *netdev; 109 struct sk_buff *rsp; 110 u32 ifindex; 111 int err; 112 113 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 114 return -EINVAL; 115 116 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 117 118 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 119 if (!rsp) 120 return -ENOMEM; 121 122 rtnl_lock(); 123 124 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 125 if (netdev) 126 err = netdev_nl_dev_fill(netdev, rsp, info); 127 else 128 err = -ENODEV; 129 130 rtnl_unlock(); 131 132 if (err) 133 goto err_free_msg; 134 135 return genlmsg_reply(rsp, info); 136 137 err_free_msg: 138 nlmsg_free(rsp); 139 return err; 140 } 141 142 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 143 { 144 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 145 struct net *net = sock_net(skb->sk); 146 struct net_device *netdev; 147 int err = 0; 148 149 rtnl_lock(); 150 for_each_netdev_dump(net, netdev, ctx->ifindex) { 151 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 152 if (err < 0) 153 break; 154 } 155 rtnl_unlock(); 156 157 return err; 158 } 159 160 static int 161 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 162 const struct genl_info *info) 163 { 164 unsigned long irq_suspend_timeout; 165 unsigned long gro_flush_timeout; 166 u32 napi_defer_hard_irqs; 167 void *hdr; 168 pid_t pid; 169 170 if (WARN_ON_ONCE(!napi->dev)) 171 return -EINVAL; 172 if (!(napi->dev->flags & IFF_UP)) 173 return 0; 174 175 hdr = genlmsg_iput(rsp, info); 176 if (!hdr) 177 return -EMSGSIZE; 178 179 if (napi->napi_id >= MIN_NAPI_ID && 180 nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 181 goto nla_put_failure; 182 183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 184 goto nla_put_failure; 185 186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 187 goto nla_put_failure; 188 189 if (napi->thread) { 190 pid = task_pid_nr(napi->thread); 191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 192 goto nla_put_failure; 193 } 194 195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 197 napi_defer_hard_irqs)) 198 goto nla_put_failure; 199 200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 202 irq_suspend_timeout)) 203 goto nla_put_failure; 204 205 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 207 gro_flush_timeout)) 208 goto nla_put_failure; 209 210 genlmsg_end(rsp, hdr); 211 212 return 0; 213 214 nla_put_failure: 215 genlmsg_cancel(rsp, hdr); 216 return -EMSGSIZE; 217 } 218 219 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 220 { 221 struct napi_struct *napi; 222 struct sk_buff *rsp; 223 u32 napi_id; 224 int err; 225 226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 227 return -EINVAL; 228 229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 230 231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 232 if (!rsp) 233 return -ENOMEM; 234 235 rtnl_lock(); 236 rcu_read_lock(); 237 238 napi = napi_by_id(napi_id); 239 if (napi) { 240 err = netdev_nl_napi_fill_one(rsp, napi, info); 241 } else { 242 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 243 err = -ENOENT; 244 } 245 246 rcu_read_unlock(); 247 rtnl_unlock(); 248 249 if (err) 250 goto err_free_msg; 251 252 return genlmsg_reply(rsp, info); 253 254 err_free_msg: 255 nlmsg_free(rsp); 256 return err; 257 } 258 259 static int 260 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 261 const struct genl_info *info, 262 struct netdev_nl_dump_ctx *ctx) 263 { 264 struct napi_struct *napi; 265 int err = 0; 266 267 if (!(netdev->flags & IFF_UP)) 268 return err; 269 270 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 271 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 272 continue; 273 274 err = netdev_nl_napi_fill_one(rsp, napi, info); 275 if (err) 276 return err; 277 ctx->napi_id = napi->napi_id; 278 } 279 return err; 280 } 281 282 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 283 { 284 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 285 const struct genl_info *info = genl_info_dump(cb); 286 struct net *net = sock_net(skb->sk); 287 struct net_device *netdev; 288 u32 ifindex = 0; 289 int err = 0; 290 291 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 292 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 293 294 rtnl_lock(); 295 if (ifindex) { 296 netdev = __dev_get_by_index(net, ifindex); 297 if (netdev) 298 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 299 else 300 err = -ENODEV; 301 } else { 302 for_each_netdev_dump(net, netdev, ctx->ifindex) { 303 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 304 if (err < 0) 305 break; 306 ctx->napi_id = 0; 307 } 308 } 309 rtnl_unlock(); 310 311 return err; 312 } 313 314 static int 315 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 316 { 317 u64 irq_suspend_timeout = 0; 318 u64 gro_flush_timeout = 0; 319 u32 defer = 0; 320 321 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 322 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 323 napi_set_defer_hard_irqs(napi, defer); 324 } 325 326 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 327 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 328 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 329 } 330 331 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 332 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 333 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 334 } 335 336 return 0; 337 } 338 339 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 340 { 341 struct napi_struct *napi; 342 unsigned int napi_id; 343 int err; 344 345 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 346 return -EINVAL; 347 348 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 349 350 rtnl_lock(); 351 rcu_read_lock(); 352 353 napi = napi_by_id(napi_id); 354 if (napi) { 355 err = netdev_nl_napi_set_config(napi, info); 356 } else { 357 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 358 err = -ENOENT; 359 } 360 361 rcu_read_unlock(); 362 rtnl_unlock(); 363 364 return err; 365 } 366 367 static int 368 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 369 u32 q_idx, u32 q_type, const struct genl_info *info) 370 { 371 struct net_devmem_dmabuf_binding *binding; 372 struct netdev_rx_queue *rxq; 373 struct netdev_queue *txq; 374 void *hdr; 375 376 hdr = genlmsg_iput(rsp, info); 377 if (!hdr) 378 return -EMSGSIZE; 379 380 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 381 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 382 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 383 goto nla_put_failure; 384 385 switch (q_type) { 386 case NETDEV_QUEUE_TYPE_RX: 387 rxq = __netif_get_rx_queue(netdev, q_idx); 388 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID, 389 rxq->napi->napi_id)) 390 goto nla_put_failure; 391 392 binding = rxq->mp_params.mp_priv; 393 if (binding && 394 nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id)) 395 goto nla_put_failure; 396 397 break; 398 case NETDEV_QUEUE_TYPE_TX: 399 txq = netdev_get_tx_queue(netdev, q_idx); 400 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID, 401 txq->napi->napi_id)) 402 goto nla_put_failure; 403 } 404 405 genlmsg_end(rsp, hdr); 406 407 return 0; 408 409 nla_put_failure: 410 genlmsg_cancel(rsp, hdr); 411 return -EMSGSIZE; 412 } 413 414 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 415 u32 q_type) 416 { 417 switch (q_type) { 418 case NETDEV_QUEUE_TYPE_RX: 419 if (q_id >= netdev->real_num_rx_queues) 420 return -EINVAL; 421 return 0; 422 case NETDEV_QUEUE_TYPE_TX: 423 if (q_id >= netdev->real_num_tx_queues) 424 return -EINVAL; 425 } 426 return 0; 427 } 428 429 static int 430 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 431 u32 q_type, const struct genl_info *info) 432 { 433 int err = 0; 434 435 if (!(netdev->flags & IFF_UP)) 436 return err; 437 438 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 439 if (err) 440 return err; 441 442 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 443 } 444 445 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 446 { 447 u32 q_id, q_type, ifindex; 448 struct net_device *netdev; 449 struct sk_buff *rsp; 450 int err; 451 452 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 453 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 454 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 455 return -EINVAL; 456 457 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 458 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 459 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 460 461 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 462 if (!rsp) 463 return -ENOMEM; 464 465 rtnl_lock(); 466 467 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 468 if (netdev) 469 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 470 else 471 err = -ENODEV; 472 473 rtnl_unlock(); 474 475 if (err) 476 goto err_free_msg; 477 478 return genlmsg_reply(rsp, info); 479 480 err_free_msg: 481 nlmsg_free(rsp); 482 return err; 483 } 484 485 static int 486 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 487 const struct genl_info *info, 488 struct netdev_nl_dump_ctx *ctx) 489 { 490 int err = 0; 491 492 if (!(netdev->flags & IFF_UP)) 493 return err; 494 495 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) { 496 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx, 497 NETDEV_QUEUE_TYPE_RX, info); 498 if (err) 499 return err; 500 } 501 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) { 502 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx, 503 NETDEV_QUEUE_TYPE_TX, info); 504 if (err) 505 return err; 506 } 507 508 return err; 509 } 510 511 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 512 { 513 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 514 const struct genl_info *info = genl_info_dump(cb); 515 struct net *net = sock_net(skb->sk); 516 struct net_device *netdev; 517 u32 ifindex = 0; 518 int err = 0; 519 520 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 521 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 522 523 rtnl_lock(); 524 if (ifindex) { 525 netdev = __dev_get_by_index(net, ifindex); 526 if (netdev) 527 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 528 else 529 err = -ENODEV; 530 } else { 531 for_each_netdev_dump(net, netdev, ctx->ifindex) { 532 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 533 if (err < 0) 534 break; 535 ctx->rxq_idx = 0; 536 ctx->txq_idx = 0; 537 } 538 } 539 rtnl_unlock(); 540 541 return err; 542 } 543 544 #define NETDEV_STAT_NOT_SET (~0ULL) 545 546 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 547 { 548 const u64 *add = _add; 549 u64 *sum = _sum; 550 551 while (size) { 552 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 553 *sum += *add; 554 sum++; 555 add++; 556 size -= 8; 557 } 558 } 559 560 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 561 { 562 if (value == NETDEV_STAT_NOT_SET) 563 return 0; 564 return nla_put_uint(rsp, attr_id, value); 565 } 566 567 static int 568 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 569 { 570 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 571 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 572 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 573 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 574 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 575 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 576 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 577 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 578 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 579 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 580 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 581 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 582 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 583 return -EMSGSIZE; 584 return 0; 585 } 586 587 static int 588 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 589 { 590 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 591 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 592 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 593 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 594 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 595 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 596 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 597 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 598 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 599 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 600 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 601 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 602 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 603 return -EMSGSIZE; 604 return 0; 605 } 606 607 static int 608 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 609 u32 q_type, int i, const struct genl_info *info) 610 { 611 const struct netdev_stat_ops *ops = netdev->stat_ops; 612 struct netdev_queue_stats_rx rx; 613 struct netdev_queue_stats_tx tx; 614 void *hdr; 615 616 hdr = genlmsg_iput(rsp, info); 617 if (!hdr) 618 return -EMSGSIZE; 619 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 620 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 621 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 622 goto nla_put_failure; 623 624 switch (q_type) { 625 case NETDEV_QUEUE_TYPE_RX: 626 memset(&rx, 0xff, sizeof(rx)); 627 ops->get_queue_stats_rx(netdev, i, &rx); 628 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 629 goto nla_cancel; 630 if (netdev_nl_stats_write_rx(rsp, &rx)) 631 goto nla_put_failure; 632 break; 633 case NETDEV_QUEUE_TYPE_TX: 634 memset(&tx, 0xff, sizeof(tx)); 635 ops->get_queue_stats_tx(netdev, i, &tx); 636 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 637 goto nla_cancel; 638 if (netdev_nl_stats_write_tx(rsp, &tx)) 639 goto nla_put_failure; 640 break; 641 } 642 643 genlmsg_end(rsp, hdr); 644 return 0; 645 646 nla_cancel: 647 genlmsg_cancel(rsp, hdr); 648 return 0; 649 nla_put_failure: 650 genlmsg_cancel(rsp, hdr); 651 return -EMSGSIZE; 652 } 653 654 static int 655 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 656 const struct genl_info *info, 657 struct netdev_nl_dump_ctx *ctx) 658 { 659 const struct netdev_stat_ops *ops = netdev->stat_ops; 660 int i, err; 661 662 if (!(netdev->flags & IFF_UP)) 663 return 0; 664 665 i = ctx->rxq_idx; 666 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 667 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 668 i, info); 669 if (err) 670 return err; 671 ctx->rxq_idx = ++i; 672 } 673 i = ctx->txq_idx; 674 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 675 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 676 i, info); 677 if (err) 678 return err; 679 ctx->txq_idx = ++i; 680 } 681 682 ctx->rxq_idx = 0; 683 ctx->txq_idx = 0; 684 return 0; 685 } 686 687 static int 688 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 689 const struct genl_info *info) 690 { 691 struct netdev_queue_stats_rx rx_sum, rx; 692 struct netdev_queue_stats_tx tx_sum, tx; 693 const struct netdev_stat_ops *ops; 694 void *hdr; 695 int i; 696 697 ops = netdev->stat_ops; 698 /* Netdev can't guarantee any complete counters */ 699 if (!ops->get_base_stats) 700 return 0; 701 702 memset(&rx_sum, 0xff, sizeof(rx_sum)); 703 memset(&tx_sum, 0xff, sizeof(tx_sum)); 704 705 ops->get_base_stats(netdev, &rx_sum, &tx_sum); 706 707 /* The op was there, but nothing reported, don't bother */ 708 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 709 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 710 return 0; 711 712 hdr = genlmsg_iput(rsp, info); 713 if (!hdr) 714 return -EMSGSIZE; 715 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 716 goto nla_put_failure; 717 718 for (i = 0; i < netdev->real_num_rx_queues; i++) { 719 memset(&rx, 0xff, sizeof(rx)); 720 if (ops->get_queue_stats_rx) 721 ops->get_queue_stats_rx(netdev, i, &rx); 722 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx)); 723 } 724 for (i = 0; i < netdev->real_num_tx_queues; i++) { 725 memset(&tx, 0xff, sizeof(tx)); 726 if (ops->get_queue_stats_tx) 727 ops->get_queue_stats_tx(netdev, i, &tx); 728 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx)); 729 } 730 731 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 732 netdev_nl_stats_write_tx(rsp, &tx_sum)) 733 goto nla_put_failure; 734 735 genlmsg_end(rsp, hdr); 736 return 0; 737 738 nla_put_failure: 739 genlmsg_cancel(rsp, hdr); 740 return -EMSGSIZE; 741 } 742 743 static int 744 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 745 struct sk_buff *skb, const struct genl_info *info, 746 struct netdev_nl_dump_ctx *ctx) 747 { 748 if (!netdev->stat_ops) 749 return 0; 750 751 switch (scope) { 752 case 0: 753 return netdev_nl_stats_by_netdev(netdev, skb, info); 754 case NETDEV_QSTATS_SCOPE_QUEUE: 755 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 756 } 757 758 return -EINVAL; /* Should not happen, per netlink policy */ 759 } 760 761 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 762 struct netlink_callback *cb) 763 { 764 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 765 const struct genl_info *info = genl_info_dump(cb); 766 struct net *net = sock_net(skb->sk); 767 struct net_device *netdev; 768 unsigned int ifindex; 769 unsigned int scope; 770 int err = 0; 771 772 scope = 0; 773 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 774 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 775 776 ifindex = 0; 777 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 778 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 779 780 rtnl_lock(); 781 if (ifindex) { 782 netdev = __dev_get_by_index(net, ifindex); 783 if (netdev && netdev->stat_ops) { 784 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 785 info, ctx); 786 } else { 787 NL_SET_BAD_ATTR(info->extack, 788 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 789 err = netdev ? -EOPNOTSUPP : -ENODEV; 790 } 791 } else { 792 for_each_netdev_dump(net, netdev, ctx->ifindex) { 793 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 794 info, ctx); 795 if (err < 0) 796 break; 797 } 798 } 799 rtnl_unlock(); 800 801 return err; 802 } 803 804 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 805 { 806 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 807 struct net_devmem_dmabuf_binding *binding; 808 struct list_head *sock_binding_list; 809 u32 ifindex, dmabuf_fd, rxq_idx; 810 struct net_device *netdev; 811 struct sk_buff *rsp; 812 struct nlattr *attr; 813 int rem, err = 0; 814 void *hdr; 815 816 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 817 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 818 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 819 return -EINVAL; 820 821 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 822 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 823 824 sock_binding_list = genl_sk_priv_get(&netdev_nl_family, 825 NETLINK_CB(skb).sk); 826 if (IS_ERR(sock_binding_list)) 827 return PTR_ERR(sock_binding_list); 828 829 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 830 if (!rsp) 831 return -ENOMEM; 832 833 hdr = genlmsg_iput(rsp, info); 834 if (!hdr) { 835 err = -EMSGSIZE; 836 goto err_genlmsg_free; 837 } 838 839 rtnl_lock(); 840 841 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 842 if (!netdev || !netif_device_present(netdev)) { 843 err = -ENODEV; 844 goto err_unlock; 845 } 846 847 if (dev_xdp_prog_count(netdev)) { 848 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached"); 849 err = -EEXIST; 850 goto err_unlock; 851 } 852 853 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack); 854 if (IS_ERR(binding)) { 855 err = PTR_ERR(binding); 856 goto err_unlock; 857 } 858 859 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 860 genlmsg_data(info->genlhdr), 861 genlmsg_len(info->genlhdr), rem) { 862 err = nla_parse_nested( 863 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr, 864 netdev_queue_id_nl_policy, info->extack); 865 if (err < 0) 866 goto err_unbind; 867 868 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 869 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) { 870 err = -EINVAL; 871 goto err_unbind; 872 } 873 874 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 875 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 876 err = -EINVAL; 877 goto err_unbind; 878 } 879 880 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 881 882 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 883 info->extack); 884 if (err) 885 goto err_unbind; 886 } 887 888 list_add(&binding->list, sock_binding_list); 889 890 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 891 genlmsg_end(rsp, hdr); 892 893 err = genlmsg_reply(rsp, info); 894 if (err) 895 goto err_unbind; 896 897 rtnl_unlock(); 898 899 return 0; 900 901 err_unbind: 902 net_devmem_unbind_dmabuf(binding); 903 err_unlock: 904 rtnl_unlock(); 905 err_genlmsg_free: 906 nlmsg_free(rsp); 907 return err; 908 } 909 910 void netdev_nl_sock_priv_init(struct list_head *priv) 911 { 912 INIT_LIST_HEAD(priv); 913 } 914 915 void netdev_nl_sock_priv_destroy(struct list_head *priv) 916 { 917 struct net_devmem_dmabuf_binding *binding; 918 struct net_devmem_dmabuf_binding *temp; 919 920 list_for_each_entry_safe(binding, temp, priv, list) { 921 rtnl_lock(); 922 net_devmem_unbind_dmabuf(binding); 923 rtnl_unlock(); 924 } 925 } 926 927 static int netdev_genl_netdevice_event(struct notifier_block *nb, 928 unsigned long event, void *ptr) 929 { 930 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 931 932 switch (event) { 933 case NETDEV_REGISTER: 934 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 935 break; 936 case NETDEV_UNREGISTER: 937 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 938 break; 939 case NETDEV_XDP_FEAT_CHANGE: 940 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 941 break; 942 } 943 944 return NOTIFY_OK; 945 } 946 947 static struct notifier_block netdev_genl_nb = { 948 .notifier_call = netdev_genl_netdevice_event, 949 }; 950 951 static int __init netdev_genl_init(void) 952 { 953 int err; 954 955 err = register_netdevice_notifier(&netdev_genl_nb); 956 if (err) 957 return err; 958 959 err = genl_register_family(&netdev_nl_family); 960 if (err) 961 goto err_unreg_ntf; 962 963 return 0; 964 965 err_unreg_ntf: 966 unregister_netdevice_notifier(&netdev_genl_nb); 967 return err; 968 } 969 970 subsys_initcall(netdev_genl_init); 971