1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/rtnetlink.h> 6 #include <net/busy_poll.h> 7 #include <net/net_namespace.h> 8 #include <net/netdev_queues.h> 9 #include <net/netdev_rx_queue.h> 10 #include <net/sock.h> 11 #include <net/xdp.h> 12 #include <net/xdp_sock.h> 13 #include <net/page_pool/memory_provider.h> 14 15 #include "dev.h" 16 #include "devmem.h" 17 #include "netdev-genl-gen.h" 18 19 struct netdev_nl_dump_ctx { 20 unsigned long ifindex; 21 unsigned int rxq_idx; 22 unsigned int txq_idx; 23 unsigned int napi_id; 24 }; 25 26 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 27 { 28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 29 30 return (struct netdev_nl_dump_ctx *)cb->ctx; 31 } 32 33 static int 34 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 35 const struct genl_info *info) 36 { 37 u64 xsk_features = 0; 38 u64 xdp_rx_meta = 0; 39 void *hdr; 40 41 hdr = genlmsg_iput(rsp, info); 42 if (!hdr) 43 return -EMSGSIZE; 44 45 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 46 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 47 xdp_rx_meta |= flag; 48 XDP_METADATA_KFUNC_xxx 49 #undef XDP_METADATA_KFUNC 50 51 if (netdev->xsk_tx_metadata_ops) { 52 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 53 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 54 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 55 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 56 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time) 57 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO; 58 } 59 60 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 61 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 62 netdev->xdp_features, NETDEV_A_DEV_PAD) || 63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 64 xdp_rx_meta, NETDEV_A_DEV_PAD) || 65 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 66 xsk_features, NETDEV_A_DEV_PAD)) 67 goto err_cancel_msg; 68 69 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 70 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 71 netdev->xdp_zc_max_segs)) 72 goto err_cancel_msg; 73 } 74 75 genlmsg_end(rsp, hdr); 76 77 return 0; 78 79 err_cancel_msg: 80 genlmsg_cancel(rsp, hdr); 81 return -EMSGSIZE; 82 } 83 84 static void 85 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 86 { 87 struct genl_info info; 88 struct sk_buff *ntf; 89 90 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 91 NETDEV_NLGRP_MGMT)) 92 return; 93 94 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 95 96 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 97 if (!ntf) 98 return; 99 100 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 101 nlmsg_free(ntf); 102 return; 103 } 104 105 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 106 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 107 } 108 109 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 110 { 111 struct net_device *netdev; 112 struct sk_buff *rsp; 113 u32 ifindex; 114 int err; 115 116 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 117 return -EINVAL; 118 119 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 120 121 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 122 if (!rsp) 123 return -ENOMEM; 124 125 rtnl_lock(); 126 127 netdev = __dev_get_by_index(genl_info_net(info), ifindex); 128 if (netdev) 129 err = netdev_nl_dev_fill(netdev, rsp, info); 130 else 131 err = -ENODEV; 132 133 rtnl_unlock(); 134 135 if (err) 136 goto err_free_msg; 137 138 return genlmsg_reply(rsp, info); 139 140 err_free_msg: 141 nlmsg_free(rsp); 142 return err; 143 } 144 145 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 146 { 147 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 148 struct net *net = sock_net(skb->sk); 149 struct net_device *netdev; 150 int err = 0; 151 152 rtnl_lock(); 153 for_each_netdev_dump(net, netdev, ctx->ifindex) { 154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 155 if (err < 0) 156 break; 157 } 158 rtnl_unlock(); 159 160 return err; 161 } 162 163 static int 164 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 165 const struct genl_info *info) 166 { 167 unsigned long irq_suspend_timeout; 168 unsigned long gro_flush_timeout; 169 u32 napi_defer_hard_irqs; 170 void *hdr; 171 pid_t pid; 172 173 if (!napi->dev->up) 174 return 0; 175 176 hdr = genlmsg_iput(rsp, info); 177 if (!hdr) 178 return -EMSGSIZE; 179 180 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 181 goto nla_put_failure; 182 183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 184 goto nla_put_failure; 185 186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 187 goto nla_put_failure; 188 189 if (napi->thread) { 190 pid = task_pid_nr(napi->thread); 191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 192 goto nla_put_failure; 193 } 194 195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 197 napi_defer_hard_irqs)) 198 goto nla_put_failure; 199 200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 202 irq_suspend_timeout)) 203 goto nla_put_failure; 204 205 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 207 gro_flush_timeout)) 208 goto nla_put_failure; 209 210 genlmsg_end(rsp, hdr); 211 212 return 0; 213 214 nla_put_failure: 215 genlmsg_cancel(rsp, hdr); 216 return -EMSGSIZE; 217 } 218 219 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 220 { 221 struct napi_struct *napi; 222 struct sk_buff *rsp; 223 u32 napi_id; 224 int err; 225 226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 227 return -EINVAL; 228 229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 230 231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 232 if (!rsp) 233 return -ENOMEM; 234 235 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 236 if (napi) { 237 err = netdev_nl_napi_fill_one(rsp, napi, info); 238 netdev_unlock(napi->dev); 239 } else { 240 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 241 err = -ENOENT; 242 } 243 244 if (err) { 245 goto err_free_msg; 246 } else if (!rsp->len) { 247 err = -ENOENT; 248 goto err_free_msg; 249 } 250 251 return genlmsg_reply(rsp, info); 252 253 err_free_msg: 254 nlmsg_free(rsp); 255 return err; 256 } 257 258 static int 259 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 260 const struct genl_info *info, 261 struct netdev_nl_dump_ctx *ctx) 262 { 263 struct napi_struct *napi; 264 unsigned int prev_id; 265 int err = 0; 266 267 if (!netdev->up) 268 return err; 269 270 prev_id = UINT_MAX; 271 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 272 if (!napi_id_valid(napi->napi_id)) 273 continue; 274 275 /* Dump continuation below depends on the list being sorted */ 276 WARN_ON_ONCE(napi->napi_id >= prev_id); 277 prev_id = napi->napi_id; 278 279 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 280 continue; 281 282 err = netdev_nl_napi_fill_one(rsp, napi, info); 283 if (err) 284 return err; 285 ctx->napi_id = napi->napi_id; 286 } 287 return err; 288 } 289 290 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 291 { 292 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 293 const struct genl_info *info = genl_info_dump(cb); 294 struct net *net = sock_net(skb->sk); 295 struct net_device *netdev; 296 u32 ifindex = 0; 297 int err = 0; 298 299 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 300 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 301 302 if (ifindex) { 303 netdev = netdev_get_by_index_lock(net, ifindex); 304 if (netdev) { 305 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 306 netdev_unlock(netdev); 307 } else { 308 err = -ENODEV; 309 } 310 } else { 311 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 312 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 313 if (err < 0) 314 break; 315 ctx->napi_id = 0; 316 } 317 } 318 319 return err; 320 } 321 322 static int 323 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 324 { 325 u64 irq_suspend_timeout = 0; 326 u64 gro_flush_timeout = 0; 327 u32 defer = 0; 328 329 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 330 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 331 napi_set_defer_hard_irqs(napi, defer); 332 } 333 334 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 335 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 336 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 337 } 338 339 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 340 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 341 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 342 } 343 344 return 0; 345 } 346 347 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 348 { 349 struct napi_struct *napi; 350 unsigned int napi_id; 351 int err; 352 353 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 354 return -EINVAL; 355 356 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 357 358 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 359 if (napi) { 360 err = netdev_nl_napi_set_config(napi, info); 361 netdev_unlock(napi->dev); 362 } else { 363 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 364 err = -ENOENT; 365 } 366 367 return err; 368 } 369 370 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi) 371 { 372 if (napi && napi_id_valid(napi->napi_id)) 373 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id); 374 return 0; 375 } 376 377 static int 378 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 379 u32 q_idx, u32 q_type, const struct genl_info *info) 380 { 381 struct pp_memory_provider_params *params; 382 struct netdev_rx_queue *rxq; 383 struct netdev_queue *txq; 384 void *hdr; 385 386 hdr = genlmsg_iput(rsp, info); 387 if (!hdr) 388 return -EMSGSIZE; 389 390 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 391 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 392 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 393 goto nla_put_failure; 394 395 switch (q_type) { 396 case NETDEV_QUEUE_TYPE_RX: 397 rxq = __netif_get_rx_queue(netdev, q_idx); 398 if (nla_put_napi_id(rsp, rxq->napi)) 399 goto nla_put_failure; 400 401 params = &rxq->mp_params; 402 if (params->mp_ops && 403 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq)) 404 goto nla_put_failure; 405 #ifdef CONFIG_XDP_SOCKETS 406 if (rxq->pool) 407 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 408 goto nla_put_failure; 409 #endif 410 411 break; 412 case NETDEV_QUEUE_TYPE_TX: 413 txq = netdev_get_tx_queue(netdev, q_idx); 414 if (nla_put_napi_id(rsp, txq->napi)) 415 goto nla_put_failure; 416 #ifdef CONFIG_XDP_SOCKETS 417 if (txq->pool) 418 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 419 goto nla_put_failure; 420 #endif 421 break; 422 } 423 424 genlmsg_end(rsp, hdr); 425 426 return 0; 427 428 nla_put_failure: 429 genlmsg_cancel(rsp, hdr); 430 return -EMSGSIZE; 431 } 432 433 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 434 u32 q_type) 435 { 436 switch (q_type) { 437 case NETDEV_QUEUE_TYPE_RX: 438 if (q_id >= netdev->real_num_rx_queues) 439 return -EINVAL; 440 return 0; 441 case NETDEV_QUEUE_TYPE_TX: 442 if (q_id >= netdev->real_num_tx_queues) 443 return -EINVAL; 444 } 445 return 0; 446 } 447 448 static int 449 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 450 u32 q_type, const struct genl_info *info) 451 { 452 int err; 453 454 if (!netdev->up) 455 return -ENOENT; 456 457 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 458 if (err) 459 return err; 460 461 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 462 } 463 464 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 465 { 466 u32 q_id, q_type, ifindex; 467 struct net_device *netdev; 468 struct sk_buff *rsp; 469 int err; 470 471 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 472 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 473 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 474 return -EINVAL; 475 476 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 477 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 478 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 479 480 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 481 if (!rsp) 482 return -ENOMEM; 483 484 rtnl_lock(); 485 486 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 487 if (netdev) { 488 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 489 netdev_unlock(netdev); 490 } else { 491 err = -ENODEV; 492 } 493 494 rtnl_unlock(); 495 496 if (err) 497 goto err_free_msg; 498 499 return genlmsg_reply(rsp, info); 500 501 err_free_msg: 502 nlmsg_free(rsp); 503 return err; 504 } 505 506 static int 507 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 508 const struct genl_info *info, 509 struct netdev_nl_dump_ctx *ctx) 510 { 511 int err = 0; 512 513 if (!netdev->up) 514 return err; 515 516 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) { 517 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx, 518 NETDEV_QUEUE_TYPE_RX, info); 519 if (err) 520 return err; 521 } 522 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) { 523 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx, 524 NETDEV_QUEUE_TYPE_TX, info); 525 if (err) 526 return err; 527 } 528 529 return err; 530 } 531 532 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 533 { 534 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 535 const struct genl_info *info = genl_info_dump(cb); 536 struct net *net = sock_net(skb->sk); 537 struct net_device *netdev; 538 u32 ifindex = 0; 539 int err = 0; 540 541 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 542 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 543 544 rtnl_lock(); 545 if (ifindex) { 546 netdev = netdev_get_by_index_lock(net, ifindex); 547 if (netdev) { 548 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 549 netdev_unlock(netdev); 550 } else { 551 err = -ENODEV; 552 } 553 } else { 554 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 555 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 556 if (err < 0) 557 break; 558 ctx->rxq_idx = 0; 559 ctx->txq_idx = 0; 560 } 561 } 562 rtnl_unlock(); 563 564 return err; 565 } 566 567 #define NETDEV_STAT_NOT_SET (~0ULL) 568 569 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 570 { 571 const u64 *add = _add; 572 u64 *sum = _sum; 573 574 while (size) { 575 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 576 *sum += *add; 577 sum++; 578 add++; 579 size -= 8; 580 } 581 } 582 583 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 584 { 585 if (value == NETDEV_STAT_NOT_SET) 586 return 0; 587 return nla_put_uint(rsp, attr_id, value); 588 } 589 590 static int 591 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 592 { 593 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 594 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 595 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 596 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 597 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 598 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) || 599 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 600 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 601 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 602 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 603 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 604 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 605 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 606 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 607 return -EMSGSIZE; 608 return 0; 609 } 610 611 static int 612 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 613 { 614 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 615 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 616 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 617 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 618 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 619 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 620 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 621 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 622 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 623 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 624 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 625 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 626 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 627 return -EMSGSIZE; 628 return 0; 629 } 630 631 static int 632 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 633 u32 q_type, int i, const struct genl_info *info) 634 { 635 const struct netdev_stat_ops *ops = netdev->stat_ops; 636 struct netdev_queue_stats_rx rx; 637 struct netdev_queue_stats_tx tx; 638 void *hdr; 639 640 hdr = genlmsg_iput(rsp, info); 641 if (!hdr) 642 return -EMSGSIZE; 643 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 644 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 645 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 646 goto nla_put_failure; 647 648 switch (q_type) { 649 case NETDEV_QUEUE_TYPE_RX: 650 memset(&rx, 0xff, sizeof(rx)); 651 ops->get_queue_stats_rx(netdev, i, &rx); 652 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 653 goto nla_cancel; 654 if (netdev_nl_stats_write_rx(rsp, &rx)) 655 goto nla_put_failure; 656 break; 657 case NETDEV_QUEUE_TYPE_TX: 658 memset(&tx, 0xff, sizeof(tx)); 659 ops->get_queue_stats_tx(netdev, i, &tx); 660 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 661 goto nla_cancel; 662 if (netdev_nl_stats_write_tx(rsp, &tx)) 663 goto nla_put_failure; 664 break; 665 } 666 667 genlmsg_end(rsp, hdr); 668 return 0; 669 670 nla_cancel: 671 genlmsg_cancel(rsp, hdr); 672 return 0; 673 nla_put_failure: 674 genlmsg_cancel(rsp, hdr); 675 return -EMSGSIZE; 676 } 677 678 static int 679 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 680 const struct genl_info *info, 681 struct netdev_nl_dump_ctx *ctx) 682 { 683 const struct netdev_stat_ops *ops = netdev->stat_ops; 684 int i, err; 685 686 if (!(netdev->flags & IFF_UP)) 687 return 0; 688 689 i = ctx->rxq_idx; 690 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 691 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 692 i, info); 693 if (err) 694 return err; 695 ctx->rxq_idx = ++i; 696 } 697 i = ctx->txq_idx; 698 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 699 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 700 i, info); 701 if (err) 702 return err; 703 ctx->txq_idx = ++i; 704 } 705 706 ctx->rxq_idx = 0; 707 ctx->txq_idx = 0; 708 return 0; 709 } 710 711 static int 712 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 713 const struct genl_info *info) 714 { 715 struct netdev_queue_stats_rx rx_sum, rx; 716 struct netdev_queue_stats_tx tx_sum, tx; 717 const struct netdev_stat_ops *ops; 718 void *hdr; 719 int i; 720 721 ops = netdev->stat_ops; 722 /* Netdev can't guarantee any complete counters */ 723 if (!ops->get_base_stats) 724 return 0; 725 726 memset(&rx_sum, 0xff, sizeof(rx_sum)); 727 memset(&tx_sum, 0xff, sizeof(tx_sum)); 728 729 ops->get_base_stats(netdev, &rx_sum, &tx_sum); 730 731 /* The op was there, but nothing reported, don't bother */ 732 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 733 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 734 return 0; 735 736 hdr = genlmsg_iput(rsp, info); 737 if (!hdr) 738 return -EMSGSIZE; 739 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 740 goto nla_put_failure; 741 742 for (i = 0; i < netdev->real_num_rx_queues; i++) { 743 memset(&rx, 0xff, sizeof(rx)); 744 if (ops->get_queue_stats_rx) 745 ops->get_queue_stats_rx(netdev, i, &rx); 746 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx)); 747 } 748 for (i = 0; i < netdev->real_num_tx_queues; i++) { 749 memset(&tx, 0xff, sizeof(tx)); 750 if (ops->get_queue_stats_tx) 751 ops->get_queue_stats_tx(netdev, i, &tx); 752 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx)); 753 } 754 755 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 756 netdev_nl_stats_write_tx(rsp, &tx_sum)) 757 goto nla_put_failure; 758 759 genlmsg_end(rsp, hdr); 760 return 0; 761 762 nla_put_failure: 763 genlmsg_cancel(rsp, hdr); 764 return -EMSGSIZE; 765 } 766 767 static int 768 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 769 struct sk_buff *skb, const struct genl_info *info, 770 struct netdev_nl_dump_ctx *ctx) 771 { 772 if (!netdev->stat_ops) 773 return 0; 774 775 switch (scope) { 776 case 0: 777 return netdev_nl_stats_by_netdev(netdev, skb, info); 778 case NETDEV_QSTATS_SCOPE_QUEUE: 779 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 780 } 781 782 return -EINVAL; /* Should not happen, per netlink policy */ 783 } 784 785 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 786 struct netlink_callback *cb) 787 { 788 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 789 const struct genl_info *info = genl_info_dump(cb); 790 struct net *net = sock_net(skb->sk); 791 struct net_device *netdev; 792 unsigned int ifindex; 793 unsigned int scope; 794 int err = 0; 795 796 scope = 0; 797 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 798 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 799 800 ifindex = 0; 801 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 802 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 803 804 rtnl_lock(); 805 if (ifindex) { 806 netdev = __dev_get_by_index(net, ifindex); 807 if (netdev && netdev->stat_ops) { 808 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 809 info, ctx); 810 } else { 811 NL_SET_BAD_ATTR(info->extack, 812 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 813 err = netdev ? -EOPNOTSUPP : -ENODEV; 814 } 815 } else { 816 for_each_netdev_dump(net, netdev, ctx->ifindex) { 817 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 818 info, ctx); 819 if (err < 0) 820 break; 821 } 822 } 823 rtnl_unlock(); 824 825 return err; 826 } 827 828 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 829 { 830 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 831 struct net_devmem_dmabuf_binding *binding; 832 u32 ifindex, dmabuf_fd, rxq_idx; 833 struct netdev_nl_sock *priv; 834 struct net_device *netdev; 835 struct sk_buff *rsp; 836 struct nlattr *attr; 837 int rem, err = 0; 838 void *hdr; 839 840 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 841 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 842 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 843 return -EINVAL; 844 845 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 846 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 847 848 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 849 if (IS_ERR(priv)) 850 return PTR_ERR(priv); 851 852 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 853 if (!rsp) 854 return -ENOMEM; 855 856 hdr = genlmsg_iput(rsp, info); 857 if (!hdr) { 858 err = -EMSGSIZE; 859 goto err_genlmsg_free; 860 } 861 862 mutex_lock(&priv->lock); 863 864 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 865 if (!netdev || !netif_device_present(netdev)) { 866 err = -ENODEV; 867 goto err_unlock_sock; 868 } 869 870 if (dev_xdp_prog_count(netdev)) { 871 NL_SET_ERR_MSG(info->extack, "unable to bind dmabuf to device with XDP program attached"); 872 err = -EEXIST; 873 goto err_unlock; 874 } 875 876 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack); 877 if (IS_ERR(binding)) { 878 err = PTR_ERR(binding); 879 goto err_unlock; 880 } 881 882 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 883 genlmsg_data(info->genlhdr), 884 genlmsg_len(info->genlhdr), rem) { 885 err = nla_parse_nested( 886 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr, 887 netdev_queue_id_nl_policy, info->extack); 888 if (err < 0) 889 goto err_unbind; 890 891 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 892 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) { 893 err = -EINVAL; 894 goto err_unbind; 895 } 896 897 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 898 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 899 err = -EINVAL; 900 goto err_unbind; 901 } 902 903 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 904 905 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 906 info->extack); 907 if (err) 908 goto err_unbind; 909 } 910 911 list_add(&binding->list, &priv->bindings); 912 913 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 914 genlmsg_end(rsp, hdr); 915 916 err = genlmsg_reply(rsp, info); 917 if (err) 918 goto err_unbind; 919 920 netdev_unlock(netdev); 921 922 mutex_unlock(&priv->lock); 923 924 return 0; 925 926 err_unbind: 927 net_devmem_unbind_dmabuf(binding); 928 err_unlock: 929 netdev_unlock(netdev); 930 err_unlock_sock: 931 mutex_unlock(&priv->lock); 932 err_genlmsg_free: 933 nlmsg_free(rsp); 934 return err; 935 } 936 937 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv) 938 { 939 INIT_LIST_HEAD(&priv->bindings); 940 mutex_init(&priv->lock); 941 } 942 943 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv) 944 { 945 struct net_devmem_dmabuf_binding *binding; 946 struct net_devmem_dmabuf_binding *temp; 947 948 mutex_lock(&priv->lock); 949 list_for_each_entry_safe(binding, temp, &priv->bindings, list) { 950 net_devmem_unbind_dmabuf(binding); 951 } 952 mutex_unlock(&priv->lock); 953 } 954 955 static int netdev_genl_netdevice_event(struct notifier_block *nb, 956 unsigned long event, void *ptr) 957 { 958 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 959 960 switch (event) { 961 case NETDEV_REGISTER: 962 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 963 break; 964 case NETDEV_UNREGISTER: 965 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 966 break; 967 case NETDEV_XDP_FEAT_CHANGE: 968 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 969 break; 970 } 971 972 return NOTIFY_OK; 973 } 974 975 static struct notifier_block netdev_genl_nb = { 976 .notifier_call = netdev_genl_netdevice_event, 977 }; 978 979 static int __init netdev_genl_init(void) 980 { 981 int err; 982 983 err = register_netdevice_notifier(&netdev_genl_nb); 984 if (err) 985 return err; 986 987 err = genl_register_family(&netdev_nl_family); 988 if (err) 989 goto err_unreg_ntf; 990 991 return 0; 992 993 err_unreg_ntf: 994 unregister_netdevice_notifier(&netdev_genl_nb); 995 return err; 996 } 997 998 subsys_initcall(netdev_genl_init); 999