1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/rtnetlink.h> 6 #include <net/busy_poll.h> 7 #include <net/net_namespace.h> 8 #include <net/netdev_queues.h> 9 #include <net/netdev_rx_queue.h> 10 #include <net/sock.h> 11 #include <net/xdp.h> 12 #include <net/xdp_sock.h> 13 #include <net/page_pool/memory_provider.h> 14 15 #include "dev.h" 16 #include "devmem.h" 17 #include "netdev-genl-gen.h" 18 19 struct netdev_nl_dump_ctx { 20 unsigned long ifindex; 21 unsigned int rxq_idx; 22 unsigned int txq_idx; 23 unsigned int napi_id; 24 }; 25 26 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 27 { 28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 29 30 return (struct netdev_nl_dump_ctx *)cb->ctx; 31 } 32 33 static int 34 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 35 const struct genl_info *info) 36 { 37 u64 xsk_features = 0; 38 u64 xdp_rx_meta = 0; 39 void *hdr; 40 41 hdr = genlmsg_iput(rsp, info); 42 if (!hdr) 43 return -EMSGSIZE; 44 45 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 46 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 47 xdp_rx_meta |= flag; 48 XDP_METADATA_KFUNC_xxx 49 #undef XDP_METADATA_KFUNC 50 51 if (netdev->xsk_tx_metadata_ops) { 52 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 53 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 54 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 55 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 56 } 57 58 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 59 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 60 netdev->xdp_features, NETDEV_A_DEV_PAD) || 61 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 62 xdp_rx_meta, NETDEV_A_DEV_PAD) || 63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 64 xsk_features, NETDEV_A_DEV_PAD)) 65 goto err_cancel_msg; 66 67 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 68 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 69 netdev->xdp_zc_max_segs)) 70 goto err_cancel_msg; 71 } 72 73 genlmsg_end(rsp, hdr); 74 75 return 0; 76 77 err_cancel_msg: 78 genlmsg_cancel(rsp, hdr); 79 return -EMSGSIZE; 80 } 81 82 static void 83 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 84 { 85 struct genl_info info; 86 struct sk_buff *ntf; 87 88 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 89 NETDEV_NLGRP_MGMT)) 90 return; 91 92 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 93 94 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 95 if (!ntf) 96 return; 97 98 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 99 nlmsg_free(ntf); 100 return; 101 } 102 103 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 104 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 105 } 106 107 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 108 { 109 struct net_device *netdev; 110 struct sk_buff *rsp; 111 u32 ifindex; 112 int err; 113 114 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 115 return -EINVAL; 116 117 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 118 119 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 120 if (!rsp) 121 return -ENOMEM; 122 123 rtnl_lock(); 124 125 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 126 if (netdev) 127 err = netdev_nl_dev_fill(netdev, rsp, info); 128 else 129 err = -ENODEV; 130 131 rtnl_unlock(); 132 133 if (err) 134 goto err_free_msg; 135 136 return genlmsg_reply(rsp, info); 137 138 err_free_msg: 139 nlmsg_free(rsp); 140 return err; 141 } 142 143 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 144 { 145 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 146 struct net *net = sock_net(skb->sk); 147 struct net_device *netdev; 148 int err = 0; 149 150 rtnl_lock(); 151 for_each_netdev_dump(net, netdev, ctx->ifindex) { 152 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 153 if (err < 0) 154 break; 155 } 156 rtnl_unlock(); 157 158 return err; 159 } 160 161 static int 162 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 163 const struct genl_info *info) 164 { 165 unsigned long irq_suspend_timeout; 166 unsigned long gro_flush_timeout; 167 u32 napi_defer_hard_irqs; 168 void *hdr; 169 pid_t pid; 170 171 if (!napi->dev->up) 172 return 0; 173 174 hdr = genlmsg_iput(rsp, info); 175 if (!hdr) 176 return -EMSGSIZE; 177 178 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 179 goto nla_put_failure; 180 181 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 182 goto nla_put_failure; 183 184 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 185 goto nla_put_failure; 186 187 if (napi->thread) { 188 pid = task_pid_nr(napi->thread); 189 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 190 goto nla_put_failure; 191 } 192 193 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 194 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 195 napi_defer_hard_irqs)) 196 goto nla_put_failure; 197 198 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 199 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 200 irq_suspend_timeout)) 201 goto nla_put_failure; 202 203 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 204 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 205 gro_flush_timeout)) 206 goto nla_put_failure; 207 208 genlmsg_end(rsp, hdr); 209 210 return 0; 211 212 nla_put_failure: 213 genlmsg_cancel(rsp, hdr); 214 return -EMSGSIZE; 215 } 216 217 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 218 { 219 struct napi_struct *napi; 220 struct sk_buff *rsp; 221 u32 napi_id; 222 int err; 223 224 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 225 return -EINVAL; 226 227 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 228 229 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 230 if (!rsp) 231 return -ENOMEM; 232 233 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 234 if (napi) { 235 err = netdev_nl_napi_fill_one(rsp, napi, info); 236 netdev_unlock(napi->dev); 237 } else { 238 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 239 err = -ENOENT; 240 } 241 242 if (err) { 243 goto err_free_msg; 244 } else if (!rsp->len) { 245 err = -ENOENT; 246 goto err_free_msg; 247 } 248 249 return genlmsg_reply(rsp, info); 250 251 err_free_msg: 252 nlmsg_free(rsp); 253 return err; 254 } 255 256 static int 257 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 258 const struct genl_info *info, 259 struct netdev_nl_dump_ctx *ctx) 260 { 261 struct napi_struct *napi; 262 unsigned int prev_id; 263 int err = 0; 264 265 if (!netdev->up) 266 return err; 267 268 prev_id = UINT_MAX; 269 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 270 if (napi->napi_id < MIN_NAPI_ID) 271 continue; 272 273 /* Dump continuation below depends on the list being sorted */ 274 WARN_ON_ONCE(napi->napi_id >= prev_id); 275 prev_id = napi->napi_id; 276 277 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 278 continue; 279 280 err = netdev_nl_napi_fill_one(rsp, napi, info); 281 if (err) 282 return err; 283 ctx->napi_id = napi->napi_id; 284 } 285 return err; 286 } 287 288 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 289 { 290 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 291 const struct genl_info *info = genl_info_dump(cb); 292 struct net *net = sock_net(skb->sk); 293 struct net_device *netdev; 294 u32 ifindex = 0; 295 int err = 0; 296 297 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 298 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 299 300 if (ifindex) { 301 netdev = netdev_get_by_index_lock(net, ifindex); 302 if (netdev) { 303 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 304 netdev_unlock(netdev); 305 } else { 306 err = -ENODEV; 307 } 308 } else { 309 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 310 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 311 if (err < 0) 312 break; 313 ctx->napi_id = 0; 314 } 315 } 316 317 return err; 318 } 319 320 static int 321 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 322 { 323 u64 irq_suspend_timeout = 0; 324 u64 gro_flush_timeout = 0; 325 u32 defer = 0; 326 327 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 328 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 329 napi_set_defer_hard_irqs(napi, defer); 330 } 331 332 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 333 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 334 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 335 } 336 337 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 338 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 339 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 340 } 341 342 return 0; 343 } 344 345 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 346 { 347 struct napi_struct *napi; 348 unsigned int napi_id; 349 int err; 350 351 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 352 return -EINVAL; 353 354 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 355 356 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 357 if (napi) { 358 err = netdev_nl_napi_set_config(napi, info); 359 netdev_unlock(napi->dev); 360 } else { 361 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 362 err = -ENOENT; 363 } 364 365 return err; 366 } 367 368 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi) 369 { 370 if (napi && napi_id_valid(napi->napi_id)) 371 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id); 372 return 0; 373 } 374 375 static int 376 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 377 u32 q_idx, u32 q_type, const struct genl_info *info) 378 { 379 struct pp_memory_provider_params *params; 380 struct netdev_rx_queue *rxq; 381 struct netdev_queue *txq; 382 void *hdr; 383 384 hdr = genlmsg_iput(rsp, info); 385 if (!hdr) 386 return -EMSGSIZE; 387 388 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 389 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 390 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 391 goto nla_put_failure; 392 393 switch (q_type) { 394 case NETDEV_QUEUE_TYPE_RX: 395 rxq = __netif_get_rx_queue(netdev, q_idx); 396 if (nla_put_napi_id(rsp, rxq->napi)) 397 goto nla_put_failure; 398 399 params = &rxq->mp_params; 400 if (params->mp_ops && 401 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq)) 402 goto nla_put_failure; 403 break; 404 case NETDEV_QUEUE_TYPE_TX: 405 txq = netdev_get_tx_queue(netdev, q_idx); 406 if (nla_put_napi_id(rsp, txq->napi)) 407 goto nla_put_failure; 408 } 409 410 genlmsg_end(rsp, hdr); 411 412 return 0; 413 414 nla_put_failure: 415 genlmsg_cancel(rsp, hdr); 416 return -EMSGSIZE; 417 } 418 419 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 420 u32 q_type) 421 { 422 switch (q_type) { 423 case NETDEV_QUEUE_TYPE_RX: 424 if (q_id >= netdev->real_num_rx_queues) 425 return -EINVAL; 426 return 0; 427 case NETDEV_QUEUE_TYPE_TX: 428 if (q_id >= netdev->real_num_tx_queues) 429 return -EINVAL; 430 } 431 return 0; 432 } 433 434 static int 435 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 436 u32 q_type, const struct genl_info *info) 437 { 438 int err; 439 440 if (!netdev->up) 441 return -ENOENT; 442 443 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 444 if (err) 445 return err; 446 447 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 448 } 449 450 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 451 { 452 u32 q_id, q_type, ifindex; 453 struct net_device *netdev; 454 struct sk_buff *rsp; 455 int err; 456 457 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 458 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 459 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 460 return -EINVAL; 461 462 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 463 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 464 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 465 466 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 467 if (!rsp) 468 return -ENOMEM; 469 470 rtnl_lock(); 471 472 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 473 if (netdev) { 474 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 475 netdev_unlock(netdev); 476 } else { 477 err = -ENODEV; 478 } 479 480 rtnl_unlock(); 481 482 if (err) 483 goto err_free_msg; 484 485 return genlmsg_reply(rsp, info); 486 487 err_free_msg: 488 nlmsg_free(rsp); 489 return err; 490 } 491 492 static int 493 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 494 const struct genl_info *info, 495 struct netdev_nl_dump_ctx *ctx) 496 { 497 int err = 0; 498 499 if (!netdev->up) 500 return err; 501 502 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) { 503 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx, 504 NETDEV_QUEUE_TYPE_RX, info); 505 if (err) 506 return err; 507 } 508 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) { 509 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx, 510 NETDEV_QUEUE_TYPE_TX, info); 511 if (err) 512 return err; 513 } 514 515 return err; 516 } 517 518 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 519 { 520 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 521 const struct genl_info *info = genl_info_dump(cb); 522 struct net *net = sock_net(skb->sk); 523 struct net_device *netdev; 524 u32 ifindex = 0; 525 int err = 0; 526 527 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 528 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 529 530 rtnl_lock(); 531 if (ifindex) { 532 netdev = netdev_get_by_index_lock(net, ifindex); 533 if (netdev) { 534 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 535 netdev_unlock(netdev); 536 } else { 537 err = -ENODEV; 538 } 539 } else { 540 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 541 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 542 if (err < 0) 543 break; 544 ctx->rxq_idx = 0; 545 ctx->txq_idx = 0; 546 } 547 } 548 rtnl_unlock(); 549 550 return err; 551 } 552 553 #define NETDEV_STAT_NOT_SET (~0ULL) 554 555 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 556 { 557 const u64 *add = _add; 558 u64 *sum = _sum; 559 560 while (size) { 561 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 562 *sum += *add; 563 sum++; 564 add++; 565 size -= 8; 566 } 567 } 568 569 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 570 { 571 if (value == NETDEV_STAT_NOT_SET) 572 return 0; 573 return nla_put_uint(rsp, attr_id, value); 574 } 575 576 static int 577 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 578 { 579 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 580 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 581 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 582 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 583 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 584 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 585 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 586 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 587 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 588 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 589 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 590 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 591 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 592 return -EMSGSIZE; 593 return 0; 594 } 595 596 static int 597 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 598 { 599 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 600 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 601 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 602 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 603 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 604 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 605 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 606 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 607 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 608 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 609 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 610 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 611 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 612 return -EMSGSIZE; 613 return 0; 614 } 615 616 static int 617 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 618 u32 q_type, int i, const struct genl_info *info) 619 { 620 const struct netdev_stat_ops *ops = netdev->stat_ops; 621 struct netdev_queue_stats_rx rx; 622 struct netdev_queue_stats_tx tx; 623 void *hdr; 624 625 hdr = genlmsg_iput(rsp, info); 626 if (!hdr) 627 return -EMSGSIZE; 628 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 629 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 630 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 631 goto nla_put_failure; 632 633 switch (q_type) { 634 case NETDEV_QUEUE_TYPE_RX: 635 memset(&rx, 0xff, sizeof(rx)); 636 ops->get_queue_stats_rx(netdev, i, &rx); 637 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 638 goto nla_cancel; 639 if (netdev_nl_stats_write_rx(rsp, &rx)) 640 goto nla_put_failure; 641 break; 642 case NETDEV_QUEUE_TYPE_TX: 643 memset(&tx, 0xff, sizeof(tx)); 644 ops->get_queue_stats_tx(netdev, i, &tx); 645 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 646 goto nla_cancel; 647 if (netdev_nl_stats_write_tx(rsp, &tx)) 648 goto nla_put_failure; 649 break; 650 } 651 652 genlmsg_end(rsp, hdr); 653 return 0; 654 655 nla_cancel: 656 genlmsg_cancel(rsp, hdr); 657 return 0; 658 nla_put_failure: 659 genlmsg_cancel(rsp, hdr); 660 return -EMSGSIZE; 661 } 662 663 static int 664 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 665 const struct genl_info *info, 666 struct netdev_nl_dump_ctx *ctx) 667 { 668 const struct netdev_stat_ops *ops = netdev->stat_ops; 669 int i, err; 670 671 if (!(netdev->flags & IFF_UP)) 672 return 0; 673 674 i = ctx->rxq_idx; 675 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 676 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 677 i, info); 678 if (err) 679 return err; 680 ctx->rxq_idx = ++i; 681 } 682 i = ctx->txq_idx; 683 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 684 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 685 i, info); 686 if (err) 687 return err; 688 ctx->txq_idx = ++i; 689 } 690 691 ctx->rxq_idx = 0; 692 ctx->txq_idx = 0; 693 return 0; 694 } 695 696 static int 697 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 698 const struct genl_info *info) 699 { 700 struct netdev_queue_stats_rx rx_sum, rx; 701 struct netdev_queue_stats_tx tx_sum, tx; 702 const struct netdev_stat_ops *ops; 703 void *hdr; 704 int i; 705 706 ops = netdev->stat_ops; 707 /* Netdev can't guarantee any complete counters */ 708 if (!ops->get_base_stats) 709 return 0; 710 711 memset(&rx_sum, 0xff, sizeof(rx_sum)); 712 memset(&tx_sum, 0xff, sizeof(tx_sum)); 713 714 ops->get_base_stats(netdev, &rx_sum, &tx_sum); 715 716 /* The op was there, but nothing reported, don't bother */ 717 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 718 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 719 return 0; 720 721 hdr = genlmsg_iput(rsp, info); 722 if (!hdr) 723 return -EMSGSIZE; 724 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 725 goto nla_put_failure; 726 727 for (i = 0; i < netdev->real_num_rx_queues; i++) { 728 memset(&rx, 0xff, sizeof(rx)); 729 if (ops->get_queue_stats_rx) 730 ops->get_queue_stats_rx(netdev, i, &rx); 731 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx)); 732 } 733 for (i = 0; i < netdev->real_num_tx_queues; i++) { 734 memset(&tx, 0xff, sizeof(tx)); 735 if (ops->get_queue_stats_tx) 736 ops->get_queue_stats_tx(netdev, i, &tx); 737 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx)); 738 } 739 740 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 741 netdev_nl_stats_write_tx(rsp, &tx_sum)) 742 goto nla_put_failure; 743 744 genlmsg_end(rsp, hdr); 745 return 0; 746 747 nla_put_failure: 748 genlmsg_cancel(rsp, hdr); 749 return -EMSGSIZE; 750 } 751 752 static int 753 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 754 struct sk_buff *skb, const struct genl_info *info, 755 struct netdev_nl_dump_ctx *ctx) 756 { 757 if (!netdev->stat_ops) 758 return 0; 759 760 switch (scope) { 761 case 0: 762 return netdev_nl_stats_by_netdev(netdev, skb, info); 763 case NETDEV_QSTATS_SCOPE_QUEUE: 764 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 765 } 766 767 return -EINVAL; /* Should not happen, per netlink policy */ 768 } 769 770 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 771 struct netlink_callback *cb) 772 { 773 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 774 const struct genl_info *info = genl_info_dump(cb); 775 struct net *net = sock_net(skb->sk); 776 struct net_device *netdev; 777 unsigned int ifindex; 778 unsigned int scope; 779 int err = 0; 780 781 scope = 0; 782 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 783 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 784 785 ifindex = 0; 786 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 787 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 788 789 rtnl_lock(); 790 if (ifindex) { 791 netdev = __dev_get_by_index(net, ifindex); 792 if (netdev && netdev->stat_ops) { 793 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 794 info, ctx); 795 } else { 796 NL_SET_BAD_ATTR(info->extack, 797 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 798 err = netdev ? -EOPNOTSUPP : -ENODEV; 799 } 800 } else { 801 for_each_netdev_dump(net, netdev, ctx->ifindex) { 802 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 803 info, ctx); 804 if (err < 0) 805 break; 806 } 807 } 808 rtnl_unlock(); 809 810 return err; 811 } 812 813 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 814 { 815 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 816 struct net_devmem_dmabuf_binding *binding; 817 struct list_head *sock_binding_list; 818 u32 ifindex, dmabuf_fd, rxq_idx; 819 struct net_device *netdev; 820 struct sk_buff *rsp; 821 struct nlattr *attr; 822 int rem, err = 0; 823 void *hdr; 824 825 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 826 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 827 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 828 return -EINVAL; 829 830 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 831 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 832 833 sock_binding_list = genl_sk_priv_get(&netdev_nl_family, 834 NETLINK_CB(skb).sk); 835 if (IS_ERR(sock_binding_list)) 836 return PTR_ERR(sock_binding_list); 837 838 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 839 if (!rsp) 840 return -ENOMEM; 841 842 hdr = genlmsg_iput(rsp, info); 843 if (!hdr) { 844 err = -EMSGSIZE; 845 goto err_genlmsg_free; 846 } 847 848 rtnl_lock(); 849 850 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 851 if (!netdev || !netif_device_present(netdev)) { 852 err = -ENODEV; 853 goto err_unlock; 854 } 855 856 if (dev_xdp_prog_count(netdev)) { 857 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached"); 858 err = -EEXIST; 859 goto err_unlock; 860 } 861 862 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack); 863 if (IS_ERR(binding)) { 864 err = PTR_ERR(binding); 865 goto err_unlock; 866 } 867 868 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 869 genlmsg_data(info->genlhdr), 870 genlmsg_len(info->genlhdr), rem) { 871 err = nla_parse_nested( 872 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr, 873 netdev_queue_id_nl_policy, info->extack); 874 if (err < 0) 875 goto err_unbind; 876 877 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 878 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) { 879 err = -EINVAL; 880 goto err_unbind; 881 } 882 883 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 884 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 885 err = -EINVAL; 886 goto err_unbind; 887 } 888 889 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 890 891 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 892 info->extack); 893 if (err) 894 goto err_unbind; 895 } 896 897 list_add(&binding->list, sock_binding_list); 898 899 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 900 genlmsg_end(rsp, hdr); 901 902 err = genlmsg_reply(rsp, info); 903 if (err) 904 goto err_unbind; 905 906 rtnl_unlock(); 907 908 return 0; 909 910 err_unbind: 911 net_devmem_unbind_dmabuf(binding); 912 err_unlock: 913 rtnl_unlock(); 914 err_genlmsg_free: 915 nlmsg_free(rsp); 916 return err; 917 } 918 919 void netdev_nl_sock_priv_init(struct list_head *priv) 920 { 921 INIT_LIST_HEAD(priv); 922 } 923 924 void netdev_nl_sock_priv_destroy(struct list_head *priv) 925 { 926 struct net_devmem_dmabuf_binding *binding; 927 struct net_devmem_dmabuf_binding *temp; 928 929 list_for_each_entry_safe(binding, temp, priv, list) { 930 rtnl_lock(); 931 net_devmem_unbind_dmabuf(binding); 932 rtnl_unlock(); 933 } 934 } 935 936 static int netdev_genl_netdevice_event(struct notifier_block *nb, 937 unsigned long event, void *ptr) 938 { 939 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 940 941 switch (event) { 942 case NETDEV_REGISTER: 943 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 944 break; 945 case NETDEV_UNREGISTER: 946 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 947 break; 948 case NETDEV_XDP_FEAT_CHANGE: 949 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 950 break; 951 } 952 953 return NOTIFY_OK; 954 } 955 956 static struct notifier_block netdev_genl_nb = { 957 .notifier_call = netdev_genl_netdevice_event, 958 }; 959 960 static int __init netdev_genl_init(void) 961 { 962 int err; 963 964 err = register_netdevice_notifier(&netdev_genl_nb); 965 if (err) 966 return err; 967 968 err = genl_register_family(&netdev_nl_family); 969 if (err) 970 goto err_unreg_ntf; 971 972 return 0; 973 974 err_unreg_ntf: 975 unregister_netdevice_notifier(&netdev_genl_nb); 976 return err; 977 } 978 979 subsys_initcall(netdev_genl_init); 980