1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/rtnetlink.h> 6 #include <net/busy_poll.h> 7 #include <net/net_namespace.h> 8 #include <net/netdev_queues.h> 9 #include <net/netdev_rx_queue.h> 10 #include <net/sock.h> 11 #include <net/xdp.h> 12 #include <net/xdp_sock.h> 13 14 #include "dev.h" 15 #include "devmem.h" 16 #include "netdev-genl-gen.h" 17 18 struct netdev_nl_dump_ctx { 19 unsigned long ifindex; 20 unsigned int rxq_idx; 21 unsigned int txq_idx; 22 unsigned int napi_id; 23 }; 24 25 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 26 { 27 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 28 29 return (struct netdev_nl_dump_ctx *)cb->ctx; 30 } 31 32 static int 33 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 34 const struct genl_info *info) 35 { 36 u64 xsk_features = 0; 37 u64 xdp_rx_meta = 0; 38 void *hdr; 39 40 hdr = genlmsg_iput(rsp, info); 41 if (!hdr) 42 return -EMSGSIZE; 43 44 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 45 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 46 xdp_rx_meta |= flag; 47 XDP_METADATA_KFUNC_xxx 48 #undef XDP_METADATA_KFUNC 49 50 if (netdev->xsk_tx_metadata_ops) { 51 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 52 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 53 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 54 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 55 } 56 57 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 58 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 59 netdev->xdp_features, NETDEV_A_DEV_PAD) || 60 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 61 xdp_rx_meta, NETDEV_A_DEV_PAD) || 62 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 63 xsk_features, NETDEV_A_DEV_PAD)) 64 goto err_cancel_msg; 65 66 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 67 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 68 netdev->xdp_zc_max_segs)) 69 goto err_cancel_msg; 70 } 71 72 genlmsg_end(rsp, hdr); 73 74 return 0; 75 76 err_cancel_msg: 77 genlmsg_cancel(rsp, hdr); 78 return -EMSGSIZE; 79 } 80 81 static void 82 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 83 { 84 struct genl_info info; 85 struct sk_buff *ntf; 86 87 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 88 NETDEV_NLGRP_MGMT)) 89 return; 90 91 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 92 93 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 94 if (!ntf) 95 return; 96 97 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 98 nlmsg_free(ntf); 99 return; 100 } 101 102 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 103 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 104 } 105 106 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 107 { 108 struct net_device *netdev; 109 struct sk_buff *rsp; 110 u32 ifindex; 111 int err; 112 113 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 114 return -EINVAL; 115 116 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 117 118 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 119 if (!rsp) 120 return -ENOMEM; 121 122 rtnl_lock(); 123 124 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 125 if (netdev) 126 err = netdev_nl_dev_fill(netdev, rsp, info); 127 else 128 err = -ENODEV; 129 130 rtnl_unlock(); 131 132 if (err) 133 goto err_free_msg; 134 135 return genlmsg_reply(rsp, info); 136 137 err_free_msg: 138 nlmsg_free(rsp); 139 return err; 140 } 141 142 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 143 { 144 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 145 struct net *net = sock_net(skb->sk); 146 struct net_device *netdev; 147 int err = 0; 148 149 rtnl_lock(); 150 for_each_netdev_dump(net, netdev, ctx->ifindex) { 151 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 152 if (err < 0) 153 break; 154 } 155 rtnl_unlock(); 156 157 return err; 158 } 159 160 static int 161 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 162 const struct genl_info *info) 163 { 164 unsigned long gro_flush_timeout; 165 u32 napi_defer_hard_irqs; 166 void *hdr; 167 pid_t pid; 168 169 if (WARN_ON_ONCE(!napi->dev)) 170 return -EINVAL; 171 if (!(napi->dev->flags & IFF_UP)) 172 return 0; 173 174 hdr = genlmsg_iput(rsp, info); 175 if (!hdr) 176 return -EMSGSIZE; 177 178 if (napi->napi_id >= MIN_NAPI_ID && 179 nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 180 goto nla_put_failure; 181 182 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 183 goto nla_put_failure; 184 185 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 186 goto nla_put_failure; 187 188 if (napi->thread) { 189 pid = task_pid_nr(napi->thread); 190 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 191 goto nla_put_failure; 192 } 193 194 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 195 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 196 napi_defer_hard_irqs)) 197 goto nla_put_failure; 198 199 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 200 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 201 gro_flush_timeout)) 202 goto nla_put_failure; 203 204 genlmsg_end(rsp, hdr); 205 206 return 0; 207 208 nla_put_failure: 209 genlmsg_cancel(rsp, hdr); 210 return -EMSGSIZE; 211 } 212 213 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 214 { 215 struct napi_struct *napi; 216 struct sk_buff *rsp; 217 u32 napi_id; 218 int err; 219 220 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 221 return -EINVAL; 222 223 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 224 225 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 226 if (!rsp) 227 return -ENOMEM; 228 229 rtnl_lock(); 230 231 napi = napi_by_id(napi_id); 232 if (napi) { 233 err = netdev_nl_napi_fill_one(rsp, napi, info); 234 } else { 235 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 236 err = -ENOENT; 237 } 238 239 rtnl_unlock(); 240 241 if (err) 242 goto err_free_msg; 243 244 return genlmsg_reply(rsp, info); 245 246 err_free_msg: 247 nlmsg_free(rsp); 248 return err; 249 } 250 251 static int 252 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 253 const struct genl_info *info, 254 struct netdev_nl_dump_ctx *ctx) 255 { 256 struct napi_struct *napi; 257 int err = 0; 258 259 if (!(netdev->flags & IFF_UP)) 260 return err; 261 262 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 263 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 264 continue; 265 266 err = netdev_nl_napi_fill_one(rsp, napi, info); 267 if (err) 268 return err; 269 ctx->napi_id = napi->napi_id; 270 } 271 return err; 272 } 273 274 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 275 { 276 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 277 const struct genl_info *info = genl_info_dump(cb); 278 struct net *net = sock_net(skb->sk); 279 struct net_device *netdev; 280 u32 ifindex = 0; 281 int err = 0; 282 283 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 284 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 285 286 rtnl_lock(); 287 if (ifindex) { 288 netdev = __dev_get_by_index(net, ifindex); 289 if (netdev) 290 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 291 else 292 err = -ENODEV; 293 } else { 294 for_each_netdev_dump(net, netdev, ctx->ifindex) { 295 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 296 if (err < 0) 297 break; 298 ctx->napi_id = 0; 299 } 300 } 301 rtnl_unlock(); 302 303 return err; 304 } 305 306 static int 307 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 308 { 309 u64 gro_flush_timeout = 0; 310 u32 defer = 0; 311 312 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 313 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 314 napi_set_defer_hard_irqs(napi, defer); 315 } 316 317 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 318 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 319 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 320 } 321 322 return 0; 323 } 324 325 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 326 { 327 struct napi_struct *napi; 328 unsigned int napi_id; 329 int err; 330 331 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 332 return -EINVAL; 333 334 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 335 336 rtnl_lock(); 337 338 napi = napi_by_id(napi_id); 339 if (napi) { 340 err = netdev_nl_napi_set_config(napi, info); 341 } else { 342 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 343 err = -ENOENT; 344 } 345 346 rtnl_unlock(); 347 348 return err; 349 } 350 351 static int 352 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 353 u32 q_idx, u32 q_type, const struct genl_info *info) 354 { 355 struct net_devmem_dmabuf_binding *binding; 356 struct netdev_rx_queue *rxq; 357 struct netdev_queue *txq; 358 void *hdr; 359 360 hdr = genlmsg_iput(rsp, info); 361 if (!hdr) 362 return -EMSGSIZE; 363 364 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 365 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 366 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 367 goto nla_put_failure; 368 369 switch (q_type) { 370 case NETDEV_QUEUE_TYPE_RX: 371 rxq = __netif_get_rx_queue(netdev, q_idx); 372 if (rxq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID, 373 rxq->napi->napi_id)) 374 goto nla_put_failure; 375 376 binding = rxq->mp_params.mp_priv; 377 if (binding && 378 nla_put_u32(rsp, NETDEV_A_QUEUE_DMABUF, binding->id)) 379 goto nla_put_failure; 380 381 break; 382 case NETDEV_QUEUE_TYPE_TX: 383 txq = netdev_get_tx_queue(netdev, q_idx); 384 if (txq->napi && nla_put_u32(rsp, NETDEV_A_QUEUE_NAPI_ID, 385 txq->napi->napi_id)) 386 goto nla_put_failure; 387 } 388 389 genlmsg_end(rsp, hdr); 390 391 return 0; 392 393 nla_put_failure: 394 genlmsg_cancel(rsp, hdr); 395 return -EMSGSIZE; 396 } 397 398 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 399 u32 q_type) 400 { 401 switch (q_type) { 402 case NETDEV_QUEUE_TYPE_RX: 403 if (q_id >= netdev->real_num_rx_queues) 404 return -EINVAL; 405 return 0; 406 case NETDEV_QUEUE_TYPE_TX: 407 if (q_id >= netdev->real_num_tx_queues) 408 return -EINVAL; 409 } 410 return 0; 411 } 412 413 static int 414 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 415 u32 q_type, const struct genl_info *info) 416 { 417 int err = 0; 418 419 if (!(netdev->flags & IFF_UP)) 420 return err; 421 422 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 423 if (err) 424 return err; 425 426 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 427 } 428 429 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 430 { 431 u32 q_id, q_type, ifindex; 432 struct net_device *netdev; 433 struct sk_buff *rsp; 434 int err; 435 436 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 437 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 438 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 439 return -EINVAL; 440 441 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 442 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 443 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 444 445 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 446 if (!rsp) 447 return -ENOMEM; 448 449 rtnl_lock(); 450 451 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 452 if (netdev) 453 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 454 else 455 err = -ENODEV; 456 457 rtnl_unlock(); 458 459 if (err) 460 goto err_free_msg; 461 462 return genlmsg_reply(rsp, info); 463 464 err_free_msg: 465 nlmsg_free(rsp); 466 return err; 467 } 468 469 static int 470 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 471 const struct genl_info *info, 472 struct netdev_nl_dump_ctx *ctx) 473 { 474 int err = 0; 475 int i; 476 477 if (!(netdev->flags & IFF_UP)) 478 return err; 479 480 for (i = ctx->rxq_idx; i < netdev->real_num_rx_queues;) { 481 err = netdev_nl_queue_fill_one(rsp, netdev, i, 482 NETDEV_QUEUE_TYPE_RX, info); 483 if (err) 484 return err; 485 ctx->rxq_idx = i++; 486 } 487 for (i = ctx->txq_idx; i < netdev->real_num_tx_queues;) { 488 err = netdev_nl_queue_fill_one(rsp, netdev, i, 489 NETDEV_QUEUE_TYPE_TX, info); 490 if (err) 491 return err; 492 ctx->txq_idx = i++; 493 } 494 495 return err; 496 } 497 498 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 499 { 500 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 501 const struct genl_info *info = genl_info_dump(cb); 502 struct net *net = sock_net(skb->sk); 503 struct net_device *netdev; 504 u32 ifindex = 0; 505 int err = 0; 506 507 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 508 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 509 510 rtnl_lock(); 511 if (ifindex) { 512 netdev = __dev_get_by_index(net, ifindex); 513 if (netdev) 514 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 515 else 516 err = -ENODEV; 517 } else { 518 for_each_netdev_dump(net, netdev, ctx->ifindex) { 519 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 520 if (err < 0) 521 break; 522 ctx->rxq_idx = 0; 523 ctx->txq_idx = 0; 524 } 525 } 526 rtnl_unlock(); 527 528 return err; 529 } 530 531 #define NETDEV_STAT_NOT_SET (~0ULL) 532 533 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 534 { 535 const u64 *add = _add; 536 u64 *sum = _sum; 537 538 while (size) { 539 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 540 *sum += *add; 541 sum++; 542 add++; 543 size -= 8; 544 } 545 } 546 547 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 548 { 549 if (value == NETDEV_STAT_NOT_SET) 550 return 0; 551 return nla_put_uint(rsp, attr_id, value); 552 } 553 554 static int 555 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 556 { 557 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 558 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 559 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 560 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 561 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 562 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 563 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 564 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 565 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 566 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 567 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 568 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 569 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 570 return -EMSGSIZE; 571 return 0; 572 } 573 574 static int 575 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 576 { 577 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 578 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 579 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 580 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 581 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 582 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 583 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 584 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 585 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 586 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 587 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 588 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 589 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 590 return -EMSGSIZE; 591 return 0; 592 } 593 594 static int 595 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 596 u32 q_type, int i, const struct genl_info *info) 597 { 598 const struct netdev_stat_ops *ops = netdev->stat_ops; 599 struct netdev_queue_stats_rx rx; 600 struct netdev_queue_stats_tx tx; 601 void *hdr; 602 603 hdr = genlmsg_iput(rsp, info); 604 if (!hdr) 605 return -EMSGSIZE; 606 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 607 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 608 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 609 goto nla_put_failure; 610 611 switch (q_type) { 612 case NETDEV_QUEUE_TYPE_RX: 613 memset(&rx, 0xff, sizeof(rx)); 614 ops->get_queue_stats_rx(netdev, i, &rx); 615 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 616 goto nla_cancel; 617 if (netdev_nl_stats_write_rx(rsp, &rx)) 618 goto nla_put_failure; 619 break; 620 case NETDEV_QUEUE_TYPE_TX: 621 memset(&tx, 0xff, sizeof(tx)); 622 ops->get_queue_stats_tx(netdev, i, &tx); 623 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 624 goto nla_cancel; 625 if (netdev_nl_stats_write_tx(rsp, &tx)) 626 goto nla_put_failure; 627 break; 628 } 629 630 genlmsg_end(rsp, hdr); 631 return 0; 632 633 nla_cancel: 634 genlmsg_cancel(rsp, hdr); 635 return 0; 636 nla_put_failure: 637 genlmsg_cancel(rsp, hdr); 638 return -EMSGSIZE; 639 } 640 641 static int 642 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 643 const struct genl_info *info, 644 struct netdev_nl_dump_ctx *ctx) 645 { 646 const struct netdev_stat_ops *ops = netdev->stat_ops; 647 int i, err; 648 649 if (!(netdev->flags & IFF_UP)) 650 return 0; 651 652 i = ctx->rxq_idx; 653 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 654 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 655 i, info); 656 if (err) 657 return err; 658 ctx->rxq_idx = i++; 659 } 660 i = ctx->txq_idx; 661 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 662 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 663 i, info); 664 if (err) 665 return err; 666 ctx->txq_idx = i++; 667 } 668 669 ctx->rxq_idx = 0; 670 ctx->txq_idx = 0; 671 return 0; 672 } 673 674 static int 675 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 676 const struct genl_info *info) 677 { 678 struct netdev_queue_stats_rx rx_sum, rx; 679 struct netdev_queue_stats_tx tx_sum, tx; 680 const struct netdev_stat_ops *ops; 681 void *hdr; 682 int i; 683 684 ops = netdev->stat_ops; 685 /* Netdev can't guarantee any complete counters */ 686 if (!ops->get_base_stats) 687 return 0; 688 689 memset(&rx_sum, 0xff, sizeof(rx_sum)); 690 memset(&tx_sum, 0xff, sizeof(tx_sum)); 691 692 ops->get_base_stats(netdev, &rx_sum, &tx_sum); 693 694 /* The op was there, but nothing reported, don't bother */ 695 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 696 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 697 return 0; 698 699 hdr = genlmsg_iput(rsp, info); 700 if (!hdr) 701 return -EMSGSIZE; 702 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 703 goto nla_put_failure; 704 705 for (i = 0; i < netdev->real_num_rx_queues; i++) { 706 memset(&rx, 0xff, sizeof(rx)); 707 if (ops->get_queue_stats_rx) 708 ops->get_queue_stats_rx(netdev, i, &rx); 709 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx)); 710 } 711 for (i = 0; i < netdev->real_num_tx_queues; i++) { 712 memset(&tx, 0xff, sizeof(tx)); 713 if (ops->get_queue_stats_tx) 714 ops->get_queue_stats_tx(netdev, i, &tx); 715 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx)); 716 } 717 718 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 719 netdev_nl_stats_write_tx(rsp, &tx_sum)) 720 goto nla_put_failure; 721 722 genlmsg_end(rsp, hdr); 723 return 0; 724 725 nla_put_failure: 726 genlmsg_cancel(rsp, hdr); 727 return -EMSGSIZE; 728 } 729 730 static int 731 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 732 struct sk_buff *skb, const struct genl_info *info, 733 struct netdev_nl_dump_ctx *ctx) 734 { 735 if (!netdev->stat_ops) 736 return 0; 737 738 switch (scope) { 739 case 0: 740 return netdev_nl_stats_by_netdev(netdev, skb, info); 741 case NETDEV_QSTATS_SCOPE_QUEUE: 742 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 743 } 744 745 return -EINVAL; /* Should not happen, per netlink policy */ 746 } 747 748 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 749 struct netlink_callback *cb) 750 { 751 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 752 const struct genl_info *info = genl_info_dump(cb); 753 struct net *net = sock_net(skb->sk); 754 struct net_device *netdev; 755 unsigned int ifindex; 756 unsigned int scope; 757 int err = 0; 758 759 scope = 0; 760 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 761 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 762 763 ifindex = 0; 764 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 765 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 766 767 rtnl_lock(); 768 if (ifindex) { 769 netdev = __dev_get_by_index(net, ifindex); 770 if (netdev && netdev->stat_ops) { 771 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 772 info, ctx); 773 } else { 774 NL_SET_BAD_ATTR(info->extack, 775 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 776 err = netdev ? -EOPNOTSUPP : -ENODEV; 777 } 778 } else { 779 for_each_netdev_dump(net, netdev, ctx->ifindex) { 780 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 781 info, ctx); 782 if (err < 0) 783 break; 784 } 785 } 786 rtnl_unlock(); 787 788 return err; 789 } 790 791 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 792 { 793 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 794 struct net_devmem_dmabuf_binding *binding; 795 struct list_head *sock_binding_list; 796 u32 ifindex, dmabuf_fd, rxq_idx; 797 struct net_device *netdev; 798 struct sk_buff *rsp; 799 struct nlattr *attr; 800 int rem, err = 0; 801 void *hdr; 802 803 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 804 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 805 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 806 return -EINVAL; 807 808 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 809 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 810 811 sock_binding_list = genl_sk_priv_get(&netdev_nl_family, 812 NETLINK_CB(skb).sk); 813 if (IS_ERR(sock_binding_list)) 814 return PTR_ERR(sock_binding_list); 815 816 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 817 if (!rsp) 818 return -ENOMEM; 819 820 hdr = genlmsg_iput(rsp, info); 821 if (!hdr) { 822 err = -EMSGSIZE; 823 goto err_genlmsg_free; 824 } 825 826 rtnl_lock(); 827 828 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 829 if (!netdev || !netif_device_present(netdev)) { 830 err = -ENODEV; 831 goto err_unlock; 832 } 833 834 if (dev_xdp_prog_count(netdev)) { 835 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached"); 836 err = -EEXIST; 837 goto err_unlock; 838 } 839 840 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack); 841 if (IS_ERR(binding)) { 842 err = PTR_ERR(binding); 843 goto err_unlock; 844 } 845 846 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 847 genlmsg_data(info->genlhdr), 848 genlmsg_len(info->genlhdr), rem) { 849 err = nla_parse_nested( 850 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr, 851 netdev_queue_id_nl_policy, info->extack); 852 if (err < 0) 853 goto err_unbind; 854 855 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 856 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) { 857 err = -EINVAL; 858 goto err_unbind; 859 } 860 861 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 862 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 863 err = -EINVAL; 864 goto err_unbind; 865 } 866 867 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 868 869 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 870 info->extack); 871 if (err) 872 goto err_unbind; 873 } 874 875 list_add(&binding->list, sock_binding_list); 876 877 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 878 genlmsg_end(rsp, hdr); 879 880 err = genlmsg_reply(rsp, info); 881 if (err) 882 goto err_unbind; 883 884 rtnl_unlock(); 885 886 return 0; 887 888 err_unbind: 889 net_devmem_unbind_dmabuf(binding); 890 err_unlock: 891 rtnl_unlock(); 892 err_genlmsg_free: 893 nlmsg_free(rsp); 894 return err; 895 } 896 897 void netdev_nl_sock_priv_init(struct list_head *priv) 898 { 899 INIT_LIST_HEAD(priv); 900 } 901 902 void netdev_nl_sock_priv_destroy(struct list_head *priv) 903 { 904 struct net_devmem_dmabuf_binding *binding; 905 struct net_devmem_dmabuf_binding *temp; 906 907 list_for_each_entry_safe(binding, temp, priv, list) { 908 rtnl_lock(); 909 net_devmem_unbind_dmabuf(binding); 910 rtnl_unlock(); 911 } 912 } 913 914 static int netdev_genl_netdevice_event(struct notifier_block *nb, 915 unsigned long event, void *ptr) 916 { 917 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 918 919 switch (event) { 920 case NETDEV_REGISTER: 921 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 922 break; 923 case NETDEV_UNREGISTER: 924 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 925 break; 926 case NETDEV_XDP_FEAT_CHANGE: 927 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 928 break; 929 } 930 931 return NOTIFY_OK; 932 } 933 934 static struct notifier_block netdev_genl_nb = { 935 .notifier_call = netdev_genl_netdevice_event, 936 }; 937 938 static int __init netdev_genl_init(void) 939 { 940 int err; 941 942 err = register_netdevice_notifier(&netdev_genl_nb); 943 if (err) 944 return err; 945 946 err = genl_register_family(&netdev_nl_family); 947 if (err) 948 goto err_unreg_ntf; 949 950 return 0; 951 952 err_unreg_ntf: 953 unregister_netdevice_notifier(&netdev_genl_nb); 954 return err; 955 } 956 957 subsys_initcall(netdev_genl_init); 958