1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/pid_namespace.h> 6 #include <linux/rtnetlink.h> 7 #include <net/busy_poll.h> 8 #include <net/net_namespace.h> 9 #include <net/netdev_queues.h> 10 #include <net/netdev_rx_queue.h> 11 #include <net/sock.h> 12 #include <net/xdp.h> 13 #include <net/xdp_sock.h> 14 #include <net/page_pool/memory_provider.h> 15 16 #include "dev.h" 17 #include "devmem.h" 18 #include "netdev-genl-gen.h" 19 20 struct netdev_nl_dump_ctx { 21 unsigned long ifindex; 22 unsigned int rxq_idx; 23 unsigned int txq_idx; 24 unsigned int napi_id; 25 }; 26 27 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 28 { 29 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 30 31 return (struct netdev_nl_dump_ctx *)cb->ctx; 32 } 33 34 static int 35 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 36 const struct genl_info *info) 37 { 38 u64 xsk_features = 0; 39 u64 xdp_rx_meta = 0; 40 void *hdr; 41 42 netdev_assert_locked(netdev); /* note: rtnl_lock may not be held! */ 43 44 hdr = genlmsg_iput(rsp, info); 45 if (!hdr) 46 return -EMSGSIZE; 47 48 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 49 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 50 xdp_rx_meta |= flag; 51 XDP_METADATA_KFUNC_xxx 52 #undef XDP_METADATA_KFUNC 53 54 if (netdev->xsk_tx_metadata_ops) { 55 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 56 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 57 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 58 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 59 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time) 60 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO; 61 } 62 63 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 64 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 65 netdev->xdp_features, NETDEV_A_DEV_PAD) || 66 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 67 xdp_rx_meta, NETDEV_A_DEV_PAD) || 68 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 69 xsk_features, NETDEV_A_DEV_PAD)) 70 goto err_cancel_msg; 71 72 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 73 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 74 netdev->xdp_zc_max_segs)) 75 goto err_cancel_msg; 76 } 77 78 genlmsg_end(rsp, hdr); 79 80 return 0; 81 82 err_cancel_msg: 83 genlmsg_cancel(rsp, hdr); 84 return -EMSGSIZE; 85 } 86 87 static void 88 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 89 { 90 struct genl_info info; 91 struct sk_buff *ntf; 92 93 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 94 NETDEV_NLGRP_MGMT)) 95 return; 96 97 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 98 99 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 100 if (!ntf) 101 return; 102 103 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 104 nlmsg_free(ntf); 105 return; 106 } 107 108 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 109 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 110 } 111 112 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 113 { 114 struct net_device *netdev; 115 struct sk_buff *rsp; 116 u32 ifindex; 117 int err; 118 119 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 120 return -EINVAL; 121 122 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 123 124 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 125 if (!rsp) 126 return -ENOMEM; 127 128 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 129 if (!netdev) { 130 err = -ENODEV; 131 goto err_free_msg; 132 } 133 134 err = netdev_nl_dev_fill(netdev, rsp, info); 135 netdev_unlock(netdev); 136 137 if (err) 138 goto err_free_msg; 139 140 return genlmsg_reply(rsp, info); 141 142 err_free_msg: 143 nlmsg_free(rsp); 144 return err; 145 } 146 147 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 148 { 149 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 150 struct net *net = sock_net(skb->sk); 151 int err; 152 153 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 155 if (err < 0) 156 return err; 157 } 158 159 return 0; 160 } 161 162 static int 163 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 164 const struct genl_info *info) 165 { 166 unsigned long irq_suspend_timeout; 167 unsigned long gro_flush_timeout; 168 u32 napi_defer_hard_irqs; 169 void *hdr; 170 pid_t pid; 171 172 if (!napi->dev->up) 173 return 0; 174 175 hdr = genlmsg_iput(rsp, info); 176 if (!hdr) 177 return -EMSGSIZE; 178 179 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 180 goto nla_put_failure; 181 182 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 183 goto nla_put_failure; 184 185 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 186 goto nla_put_failure; 187 188 if (nla_put_uint(rsp, NETDEV_A_NAPI_THREADED, 189 napi_get_threaded(napi))) 190 goto nla_put_failure; 191 192 if (napi->thread) { 193 pid = task_pid_nr_ns(napi->thread, 194 task_active_pid_ns(current)); 195 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 196 goto nla_put_failure; 197 } 198 199 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 200 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 201 napi_defer_hard_irqs)) 202 goto nla_put_failure; 203 204 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 205 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 206 irq_suspend_timeout)) 207 goto nla_put_failure; 208 209 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 210 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 211 gro_flush_timeout)) 212 goto nla_put_failure; 213 214 genlmsg_end(rsp, hdr); 215 216 return 0; 217 218 nla_put_failure: 219 genlmsg_cancel(rsp, hdr); 220 return -EMSGSIZE; 221 } 222 223 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 224 { 225 struct napi_struct *napi; 226 struct sk_buff *rsp; 227 u32 napi_id; 228 int err; 229 230 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 231 return -EINVAL; 232 233 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 234 235 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 236 if (!rsp) 237 return -ENOMEM; 238 239 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 240 if (napi) { 241 err = netdev_nl_napi_fill_one(rsp, napi, info); 242 netdev_unlock(napi->dev); 243 } else { 244 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 245 err = -ENOENT; 246 } 247 248 if (err) { 249 goto err_free_msg; 250 } else if (!rsp->len) { 251 err = -ENOENT; 252 goto err_free_msg; 253 } 254 255 return genlmsg_reply(rsp, info); 256 257 err_free_msg: 258 nlmsg_free(rsp); 259 return err; 260 } 261 262 static int 263 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 264 const struct genl_info *info, 265 struct netdev_nl_dump_ctx *ctx) 266 { 267 struct napi_struct *napi; 268 unsigned int prev_id; 269 int err = 0; 270 271 if (!netdev->up) 272 return err; 273 274 prev_id = UINT_MAX; 275 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 276 if (!napi_id_valid(napi->napi_id)) 277 continue; 278 279 /* Dump continuation below depends on the list being sorted */ 280 WARN_ON_ONCE(napi->napi_id >= prev_id); 281 prev_id = napi->napi_id; 282 283 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 284 continue; 285 286 err = netdev_nl_napi_fill_one(rsp, napi, info); 287 if (err) 288 return err; 289 ctx->napi_id = napi->napi_id; 290 } 291 return err; 292 } 293 294 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 295 { 296 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 297 const struct genl_info *info = genl_info_dump(cb); 298 struct net *net = sock_net(skb->sk); 299 struct net_device *netdev; 300 u32 ifindex = 0; 301 int err = 0; 302 303 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 304 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 305 306 if (ifindex) { 307 netdev = netdev_get_by_index_lock(net, ifindex); 308 if (netdev) { 309 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 310 netdev_unlock(netdev); 311 } else { 312 err = -ENODEV; 313 } 314 } else { 315 unsigned long start_ifindex = ctx->ifindex; 316 317 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 318 if (ctx->ifindex != start_ifindex) 319 ctx->napi_id = 0; 320 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 321 if (err < 0) 322 break; 323 } 324 } 325 326 return err; 327 } 328 329 static int 330 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 331 { 332 u64 irq_suspend_timeout = 0; 333 u64 gro_flush_timeout = 0; 334 u8 threaded = 0; 335 u32 defer = 0; 336 337 if (info->attrs[NETDEV_A_NAPI_THREADED]) { 338 int ret; 339 340 threaded = nla_get_uint(info->attrs[NETDEV_A_NAPI_THREADED]); 341 ret = napi_set_threaded(napi, threaded); 342 if (ret) 343 return ret; 344 } 345 346 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 347 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 348 napi_set_defer_hard_irqs(napi, defer); 349 } 350 351 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 352 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 353 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 354 } 355 356 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 357 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 358 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 359 } 360 361 return 0; 362 } 363 364 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 365 { 366 struct napi_struct *napi; 367 unsigned int napi_id; 368 int err; 369 370 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 371 return -EINVAL; 372 373 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 374 375 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 376 if (napi) { 377 err = netdev_nl_napi_set_config(napi, info); 378 netdev_unlock(napi->dev); 379 } else { 380 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 381 err = -ENOENT; 382 } 383 384 return err; 385 } 386 387 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi) 388 { 389 if (napi && napi_id_valid(napi->napi_id)) 390 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id); 391 return 0; 392 } 393 394 static int 395 netdev_nl_queue_fill_lease(struct sk_buff *rsp, struct net_device *netdev, 396 u32 q_idx, u32 q_type) 397 { 398 struct net_device *orig_netdev = netdev; 399 struct nlattr *nest_lease, *nest_queue; 400 struct netdev_rx_queue *rxq; 401 struct net *net, *peer_net; 402 403 rxq = __netif_get_rx_queue_lease(&netdev, &q_idx, NETIF_PHYS_TO_VIRT); 404 if (!rxq || orig_netdev == netdev) 405 return 0; 406 407 nest_lease = nla_nest_start(rsp, NETDEV_A_QUEUE_LEASE); 408 if (!nest_lease) 409 goto nla_put_failure; 410 411 nest_queue = nla_nest_start(rsp, NETDEV_A_LEASE_QUEUE); 412 if (!nest_queue) 413 goto nla_put_failure; 414 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx)) 415 goto nla_put_failure; 416 if (nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type)) 417 goto nla_put_failure; 418 nla_nest_end(rsp, nest_queue); 419 420 if (nla_put_u32(rsp, NETDEV_A_LEASE_IFINDEX, 421 READ_ONCE(netdev->ifindex))) 422 goto nla_put_failure; 423 424 rcu_read_lock(); 425 peer_net = dev_net_rcu(netdev); 426 net = dev_net_rcu(orig_netdev); 427 if (!net_eq(net, peer_net)) { 428 s32 id = peernet2id_alloc(net, peer_net, GFP_ATOMIC); 429 430 if (nla_put_s32(rsp, NETDEV_A_LEASE_NETNS_ID, id)) 431 goto nla_put_failure_unlock; 432 } 433 rcu_read_unlock(); 434 nla_nest_end(rsp, nest_lease); 435 return 0; 436 437 nla_put_failure_unlock: 438 rcu_read_unlock(); 439 nla_put_failure: 440 return -EMSGSIZE; 441 } 442 443 static int 444 __netdev_nl_queue_fill_mp(struct sk_buff *rsp, struct netdev_rx_queue *rxq) 445 { 446 struct pp_memory_provider_params *params = &rxq->mp_params; 447 448 if (params->mp_ops && 449 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq)) 450 return -EMSGSIZE; 451 452 #ifdef CONFIG_XDP_SOCKETS 453 if (rxq->pool) 454 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 455 return -EMSGSIZE; 456 #endif 457 return 0; 458 } 459 460 static int 461 netdev_nl_queue_fill_mp(struct sk_buff *rsp, struct net_device *netdev, 462 struct netdev_rx_queue *rxq) 463 { 464 struct netdev_rx_queue *hw_rxq; 465 int ret; 466 467 hw_rxq = rxq->lease; 468 if (!hw_rxq || !netif_is_queue_leasee(netdev)) 469 return __netdev_nl_queue_fill_mp(rsp, rxq); 470 471 netdev_lock(hw_rxq->dev); 472 ret = __netdev_nl_queue_fill_mp(rsp, hw_rxq); 473 netdev_unlock(hw_rxq->dev); 474 return ret; 475 } 476 477 static int 478 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 479 u32 q_idx, u32 q_type, const struct genl_info *info) 480 { 481 struct netdev_rx_queue *rxq; 482 struct netdev_queue *txq; 483 void *hdr; 484 485 hdr = genlmsg_iput(rsp, info); 486 if (!hdr) 487 return -EMSGSIZE; 488 489 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 490 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 491 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 492 goto nla_put_failure; 493 494 switch (q_type) { 495 case NETDEV_QUEUE_TYPE_RX: 496 rxq = __netif_get_rx_queue(netdev, q_idx); 497 if (nla_put_napi_id(rsp, rxq->napi)) 498 goto nla_put_failure; 499 if (netdev_nl_queue_fill_lease(rsp, netdev, q_idx, q_type)) 500 goto nla_put_failure; 501 if (netdev_nl_queue_fill_mp(rsp, netdev, rxq)) 502 goto nla_put_failure; 503 break; 504 case NETDEV_QUEUE_TYPE_TX: 505 txq = netdev_get_tx_queue(netdev, q_idx); 506 if (nla_put_napi_id(rsp, txq->napi)) 507 goto nla_put_failure; 508 #ifdef CONFIG_XDP_SOCKETS 509 if (txq->pool) 510 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 511 goto nla_put_failure; 512 #endif 513 break; 514 } 515 516 genlmsg_end(rsp, hdr); 517 518 return 0; 519 520 nla_put_failure: 521 genlmsg_cancel(rsp, hdr); 522 return -EMSGSIZE; 523 } 524 525 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 526 u32 q_type) 527 { 528 switch (q_type) { 529 case NETDEV_QUEUE_TYPE_RX: 530 if (q_id >= netdev->real_num_rx_queues) 531 return -EINVAL; 532 return 0; 533 case NETDEV_QUEUE_TYPE_TX: 534 if (q_id >= netdev->real_num_tx_queues) 535 return -EINVAL; 536 } 537 return 0; 538 } 539 540 static int 541 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 542 u32 q_type, const struct genl_info *info) 543 { 544 int err; 545 546 if (!netdev->up) 547 return -ENOENT; 548 549 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 550 if (err) 551 return err; 552 553 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 554 } 555 556 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 557 { 558 u32 q_id, q_type, ifindex; 559 struct net_device *netdev; 560 struct sk_buff *rsp; 561 int err; 562 563 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 564 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 565 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 566 return -EINVAL; 567 568 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 569 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 570 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 571 572 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 573 if (!rsp) 574 return -ENOMEM; 575 576 netdev = netdev_get_by_index_lock_ops_compat(genl_info_net(info), 577 ifindex); 578 if (netdev) { 579 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 580 netdev_unlock_ops_compat(netdev); 581 } else { 582 err = -ENODEV; 583 } 584 585 if (err) 586 goto err_free_msg; 587 588 return genlmsg_reply(rsp, info); 589 590 err_free_msg: 591 nlmsg_free(rsp); 592 return err; 593 } 594 595 static int 596 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 597 const struct genl_info *info, 598 struct netdev_nl_dump_ctx *ctx) 599 { 600 int err = 0; 601 602 if (!netdev->up) 603 return err; 604 605 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) { 606 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx, 607 NETDEV_QUEUE_TYPE_RX, info); 608 if (err) 609 return err; 610 } 611 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) { 612 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx, 613 NETDEV_QUEUE_TYPE_TX, info); 614 if (err) 615 return err; 616 } 617 618 return err; 619 } 620 621 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 622 { 623 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 624 const struct genl_info *info = genl_info_dump(cb); 625 struct net *net = sock_net(skb->sk); 626 struct net_device *netdev; 627 u32 ifindex = 0; 628 int err = 0; 629 630 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 631 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 632 633 if (ifindex) { 634 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex); 635 if (netdev) { 636 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 637 netdev_unlock_ops_compat(netdev); 638 } else { 639 err = -ENODEV; 640 } 641 } else { 642 unsigned long start_ifindex = ctx->ifindex; 643 644 for_each_netdev_lock_ops_compat_scoped(net, netdev, 645 ctx->ifindex) { 646 if (ctx->ifindex != start_ifindex) { 647 ctx->rxq_idx = 0; 648 ctx->txq_idx = 0; 649 } 650 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 651 if (err < 0) 652 break; 653 } 654 } 655 656 return err; 657 } 658 659 #define NETDEV_STAT_NOT_SET (~0ULL) 660 661 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 662 { 663 const u64 *add = _add; 664 u64 *sum = _sum; 665 666 while (size) { 667 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 668 *sum += *add; 669 sum++; 670 add++; 671 size -= 8; 672 } 673 } 674 675 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 676 { 677 if (value == NETDEV_STAT_NOT_SET) 678 return 0; 679 return nla_put_uint(rsp, attr_id, value); 680 } 681 682 static int 683 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 684 { 685 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 686 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 687 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 688 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 689 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 690 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) || 691 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 692 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 693 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 694 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 695 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 696 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 697 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 698 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 699 return -EMSGSIZE; 700 return 0; 701 } 702 703 static int 704 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 705 { 706 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 707 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 708 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 709 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 710 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 711 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 712 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 713 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 714 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 715 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 716 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 717 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 718 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 719 return -EMSGSIZE; 720 return 0; 721 } 722 723 static int 724 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 725 u32 q_type, int i, const struct genl_info *info) 726 { 727 const struct netdev_stat_ops *ops = netdev->stat_ops; 728 struct netdev_queue_stats_rx rx; 729 struct netdev_queue_stats_tx tx; 730 void *hdr; 731 732 hdr = genlmsg_iput(rsp, info); 733 if (!hdr) 734 return -EMSGSIZE; 735 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 736 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 737 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 738 goto nla_put_failure; 739 740 switch (q_type) { 741 case NETDEV_QUEUE_TYPE_RX: 742 memset(&rx, 0xff, sizeof(rx)); 743 ops->get_queue_stats_rx(netdev, i, &rx); 744 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 745 goto nla_cancel; 746 if (netdev_nl_stats_write_rx(rsp, &rx)) 747 goto nla_put_failure; 748 break; 749 case NETDEV_QUEUE_TYPE_TX: 750 memset(&tx, 0xff, sizeof(tx)); 751 ops->get_queue_stats_tx(netdev, i, &tx); 752 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 753 goto nla_cancel; 754 if (netdev_nl_stats_write_tx(rsp, &tx)) 755 goto nla_put_failure; 756 break; 757 } 758 759 genlmsg_end(rsp, hdr); 760 return 0; 761 762 nla_cancel: 763 genlmsg_cancel(rsp, hdr); 764 return 0; 765 nla_put_failure: 766 genlmsg_cancel(rsp, hdr); 767 return -EMSGSIZE; 768 } 769 770 static int 771 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 772 const struct genl_info *info, 773 struct netdev_nl_dump_ctx *ctx) 774 { 775 const struct netdev_stat_ops *ops = netdev->stat_ops; 776 int i, err; 777 778 if (!netdev->up) 779 return 0; 780 781 i = ctx->rxq_idx; 782 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 783 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 784 i, info); 785 if (err) 786 return err; 787 ctx->rxq_idx = ++i; 788 } 789 i = ctx->txq_idx; 790 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 791 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 792 i, info); 793 if (err) 794 return err; 795 ctx->txq_idx = ++i; 796 } 797 798 return 0; 799 } 800 801 /** 802 * netdev_stat_queue_sum() - add up queue stats from range of queues 803 * @netdev: net_device 804 * @rx_start: index of the first Rx queue to query 805 * @rx_end: index after the last Rx queue (first *not* to query) 806 * @rx_sum: output Rx stats, should be already initialized 807 * @tx_start: index of the first Tx queue to query 808 * @tx_end: index after the last Tx queue (first *not* to query) 809 * @tx_sum: output Tx stats, should be already initialized 810 * 811 * Add stats from [start, end) range of queue IDs to *x_sum structs. 812 * The sum structs must be already initialized. Usually this 813 * helper is invoked from the .get_base_stats callbacks of drivers 814 * to account for stats of disabled queues. In that case the ranges 815 * are usually [netdev->real_num_*x_queues, netdev->num_*x_queues). 816 */ 817 void netdev_stat_queue_sum(struct net_device *netdev, 818 int rx_start, int rx_end, 819 struct netdev_queue_stats_rx *rx_sum, 820 int tx_start, int tx_end, 821 struct netdev_queue_stats_tx *tx_sum) 822 { 823 const struct netdev_stat_ops *ops; 824 struct netdev_queue_stats_rx rx; 825 struct netdev_queue_stats_tx tx; 826 int i; 827 828 ops = netdev->stat_ops; 829 830 for (i = rx_start; i < rx_end; i++) { 831 memset(&rx, 0xff, sizeof(rx)); 832 if (ops->get_queue_stats_rx) 833 ops->get_queue_stats_rx(netdev, i, &rx); 834 netdev_nl_stats_add(rx_sum, &rx, sizeof(rx)); 835 } 836 for (i = tx_start; i < tx_end; i++) { 837 memset(&tx, 0xff, sizeof(tx)); 838 if (ops->get_queue_stats_tx) 839 ops->get_queue_stats_tx(netdev, i, &tx); 840 netdev_nl_stats_add(tx_sum, &tx, sizeof(tx)); 841 } 842 } 843 EXPORT_SYMBOL(netdev_stat_queue_sum); 844 845 static int 846 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 847 const struct genl_info *info) 848 { 849 struct netdev_queue_stats_rx rx_sum; 850 struct netdev_queue_stats_tx tx_sum; 851 void *hdr; 852 853 /* Netdev can't guarantee any complete counters */ 854 if (!netdev->stat_ops->get_base_stats) 855 return 0; 856 857 memset(&rx_sum, 0xff, sizeof(rx_sum)); 858 memset(&tx_sum, 0xff, sizeof(tx_sum)); 859 860 netdev->stat_ops->get_base_stats(netdev, &rx_sum, &tx_sum); 861 862 /* The op was there, but nothing reported, don't bother */ 863 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 864 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 865 return 0; 866 867 hdr = genlmsg_iput(rsp, info); 868 if (!hdr) 869 return -EMSGSIZE; 870 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 871 goto nla_put_failure; 872 873 netdev_stat_queue_sum(netdev, 0, netdev->real_num_rx_queues, &rx_sum, 874 0, netdev->real_num_tx_queues, &tx_sum); 875 876 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 877 netdev_nl_stats_write_tx(rsp, &tx_sum)) 878 goto nla_put_failure; 879 880 genlmsg_end(rsp, hdr); 881 return 0; 882 883 nla_put_failure: 884 genlmsg_cancel(rsp, hdr); 885 return -EMSGSIZE; 886 } 887 888 static int 889 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 890 struct sk_buff *skb, const struct genl_info *info, 891 struct netdev_nl_dump_ctx *ctx) 892 { 893 if (!netdev->stat_ops) 894 return 0; 895 896 switch (scope) { 897 case 0: 898 return netdev_nl_stats_by_netdev(netdev, skb, info); 899 case NETDEV_QSTATS_SCOPE_QUEUE: 900 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 901 } 902 903 return -EINVAL; /* Should not happen, per netlink policy */ 904 } 905 906 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 907 struct netlink_callback *cb) 908 { 909 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 910 const struct genl_info *info = genl_info_dump(cb); 911 struct net *net = sock_net(skb->sk); 912 unsigned long start_ifindex; 913 struct net_device *netdev; 914 unsigned int ifindex; 915 unsigned int scope; 916 int err = 0; 917 918 scope = 0; 919 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 920 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 921 922 ifindex = 0; 923 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 924 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 925 926 if (ifindex) { 927 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex); 928 if (!netdev) { 929 NL_SET_BAD_ATTR(info->extack, 930 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 931 return -ENODEV; 932 } 933 if (netdev->stat_ops) { 934 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 935 info, ctx); 936 } else { 937 NL_SET_BAD_ATTR(info->extack, 938 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 939 err = -EOPNOTSUPP; 940 } 941 netdev_unlock_ops_compat(netdev); 942 return err; 943 } 944 945 start_ifindex = ctx->ifindex; 946 947 for_each_netdev_lock_ops_compat_scoped(net, netdev, ctx->ifindex) { 948 if (ctx->ifindex != start_ifindex) { 949 ctx->rxq_idx = 0; 950 ctx->txq_idx = 0; 951 } 952 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 953 info, ctx); 954 if (err < 0) 955 break; 956 } 957 958 return err; 959 } 960 961 static int netdev_nl_read_rxq_bitmap(struct genl_info *info, 962 u32 rxq_bitmap_len, 963 unsigned long *rxq_bitmap) 964 { 965 const int maxtype = ARRAY_SIZE(netdev_queue_id_nl_policy) - 1; 966 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 967 struct nlattr *attr; 968 int rem, err = 0; 969 u32 rxq_idx; 970 971 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 972 genlmsg_data(info->genlhdr), 973 genlmsg_len(info->genlhdr), rem) { 974 err = nla_parse_nested(tb, maxtype, attr, 975 netdev_queue_id_nl_policy, info->extack); 976 if (err < 0) 977 return err; 978 979 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 980 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) 981 return -EINVAL; 982 983 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 984 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 985 return -EINVAL; 986 } 987 988 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 989 if (rxq_idx >= rxq_bitmap_len) { 990 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_ID]); 991 return -EINVAL; 992 } 993 994 bitmap_set(rxq_bitmap, rxq_idx, 1); 995 } 996 997 return 0; 998 } 999 1000 static struct device * 1001 netdev_nl_get_dma_dev(struct net_device *netdev, unsigned long *rxq_bitmap, 1002 struct netlink_ext_ack *extack) 1003 { 1004 struct device *dma_dev = NULL; 1005 u32 rxq_idx, prev_rxq_idx; 1006 1007 for_each_set_bit(rxq_idx, rxq_bitmap, netdev->real_num_rx_queues) { 1008 struct device *rxq_dma_dev; 1009 1010 rxq_dma_dev = netdev_queue_get_dma_dev(netdev, rxq_idx, 1011 NETDEV_QUEUE_TYPE_RX); 1012 if (dma_dev && rxq_dma_dev != dma_dev) { 1013 NL_SET_ERR_MSG_FMT(extack, "DMA device mismatch between queue %u and %u (multi-PF device?)", 1014 rxq_idx, prev_rxq_idx); 1015 return ERR_PTR(-EOPNOTSUPP); 1016 } 1017 1018 dma_dev = rxq_dma_dev; 1019 prev_rxq_idx = rxq_idx; 1020 } 1021 1022 return dma_dev; 1023 } 1024 1025 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 1026 { 1027 struct net_devmem_dmabuf_binding *binding; 1028 unsigned int niov_shift = PAGE_SHIFT; 1029 u32 ifindex, dmabuf_fd, rxq_idx; 1030 struct netdev_nl_sock *priv; 1031 struct net_device *netdev; 1032 unsigned long *rxq_bitmap; 1033 struct device *dma_dev; 1034 struct sk_buff *rsp; 1035 int err = 0; 1036 void *hdr; 1037 1038 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 1039 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 1040 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 1041 return -EINVAL; 1042 1043 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 1044 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 1045 1046 if (info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]) { 1047 u32 rx_page_size = nla_get_u32(info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]); 1048 1049 if (!is_power_of_2(rx_page_size)) { 1050 NL_SET_ERR_MSG_ATTR(info->extack, 1051 info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE], 1052 "rx-page-size must be a power of 2"); 1053 return -EINVAL; 1054 } 1055 niov_shift = ilog2(rx_page_size); 1056 } 1057 1058 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 1059 if (IS_ERR(priv)) 1060 return PTR_ERR(priv); 1061 1062 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 1063 if (!rsp) 1064 return -ENOMEM; 1065 1066 hdr = genlmsg_iput(rsp, info); 1067 if (!hdr) { 1068 err = -EMSGSIZE; 1069 goto err_genlmsg_free; 1070 } 1071 1072 mutex_lock(&priv->lock); 1073 1074 err = 0; 1075 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1076 if (!netdev) { 1077 err = -ENODEV; 1078 goto err_unlock_sock; 1079 } 1080 if (!netif_device_present(netdev)) 1081 err = -ENODEV; 1082 else if (!netdev_need_ops_lock(netdev)) 1083 err = -EOPNOTSUPP; 1084 if (err) { 1085 NL_SET_BAD_ATTR(info->extack, 1086 info->attrs[NETDEV_A_DEV_IFINDEX]); 1087 goto err_unlock; 1088 } 1089 1090 rxq_bitmap = bitmap_zalloc(netdev->real_num_rx_queues, GFP_KERNEL); 1091 if (!rxq_bitmap) { 1092 err = -ENOMEM; 1093 goto err_unlock; 1094 } 1095 1096 err = netdev_nl_read_rxq_bitmap(info, netdev->real_num_rx_queues, 1097 rxq_bitmap); 1098 if (err) 1099 goto err_rxq_bitmap; 1100 1101 dma_dev = netdev_nl_get_dma_dev(netdev, rxq_bitmap, info->extack); 1102 if (IS_ERR(dma_dev)) { 1103 err = PTR_ERR(dma_dev); 1104 goto err_rxq_bitmap; 1105 } 1106 1107 binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE, 1108 dmabuf_fd, niov_shift, priv, 1109 info->extack); 1110 if (IS_ERR(binding)) { 1111 err = PTR_ERR(binding); 1112 goto err_rxq_bitmap; 1113 } 1114 1115 for_each_set_bit(rxq_idx, rxq_bitmap, netdev->real_num_rx_queues) { 1116 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 1117 info->extack); 1118 if (err) 1119 goto err_unbind; 1120 } 1121 1122 /* rsp was allocated large enough */ 1123 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id)); 1124 1125 genlmsg_end(rsp, hdr); 1126 1127 err = genlmsg_reply(rsp, info); 1128 1129 bitmap_free(rxq_bitmap); 1130 1131 netdev_unlock(netdev); 1132 1133 mutex_unlock(&priv->lock); 1134 1135 return err < 0 ? err : 0; 1136 1137 err_unbind: 1138 net_devmem_unbind_dmabuf(binding); 1139 err_rxq_bitmap: 1140 bitmap_free(rxq_bitmap); 1141 err_unlock: 1142 netdev_unlock(netdev); 1143 err_unlock_sock: 1144 mutex_unlock(&priv->lock); 1145 err_genlmsg_free: 1146 nlmsg_free(rsp); 1147 return err; 1148 } 1149 1150 /* Find the DMA-capable device for a netmem TX binding. 1151 * 1152 * For NETMEM_TX_DMA devices, return the device itself. 1153 * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying 1154 * physical device and return it. 1155 */ 1156 static struct net_device * 1157 netdev_find_netmem_tx_dev(struct net_device *dev) 1158 { 1159 struct netdev_rx_queue *lease_rxq; 1160 struct net_device *phys_dev; 1161 int i; 1162 1163 if (dev->netmem_tx == NETMEM_TX_DMA) 1164 return dev; 1165 1166 if (dev->netmem_tx != NETMEM_TX_NO_DMA) 1167 return NULL; 1168 1169 for (i = 0; i < dev->real_num_rx_queues; i++) { 1170 lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease); 1171 if (!lease_rxq) 1172 continue; 1173 1174 phys_dev = lease_rxq->dev; 1175 if (netif_device_present(phys_dev) && 1176 phys_dev->netmem_tx == NETMEM_TX_DMA) 1177 return phys_dev; 1178 } 1179 1180 return NULL; 1181 } 1182 1183 /* Note: NETDEV_CMD_BIND_TX is intentionally unprivileged (no 1184 * GENL_ADMIN_PERM / GENL_UNS_ADMIN_PERM). Unlike bind-rx, which configures 1185 * shared NIC RX queues, bind-tx only DMA-maps the caller's dmabuf so they can 1186 * transmit from it on their own sockets without affecting other traffic or 1187 * device state. 1188 */ 1189 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) 1190 { 1191 struct net_devmem_dmabuf_binding *binding; 1192 struct net_device *bind_dev; 1193 struct netdev_nl_sock *priv; 1194 struct net_device *netdev; 1195 struct device *dma_dev; 1196 u32 ifindex, dmabuf_fd; 1197 struct sk_buff *rsp; 1198 int err = 0; 1199 void *hdr; 1200 1201 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 1202 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD)) 1203 return -EINVAL; 1204 1205 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 1206 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 1207 1208 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 1209 if (IS_ERR(priv)) 1210 return PTR_ERR(priv); 1211 1212 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 1213 if (!rsp) 1214 return -ENOMEM; 1215 1216 hdr = genlmsg_iput(rsp, info); 1217 if (!hdr) { 1218 err = -EMSGSIZE; 1219 goto err_genlmsg_free; 1220 } 1221 1222 mutex_lock(&priv->lock); 1223 1224 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1225 if (!netdev) { 1226 err = -ENODEV; 1227 goto err_unlock_sock; 1228 } 1229 1230 if (!netif_device_present(netdev)) { 1231 err = -ENODEV; 1232 goto err_unlock_netdev; 1233 } 1234 1235 if (netdev->netmem_tx == NETMEM_TX_NONE) { 1236 err = -EOPNOTSUPP; 1237 NL_SET_ERR_MSG(info->extack, 1238 "Driver does not support netmem TX"); 1239 goto err_unlock_netdev; 1240 } 1241 1242 bind_dev = netdev_find_netmem_tx_dev(netdev); 1243 if (!bind_dev) { 1244 err = -EOPNOTSUPP; 1245 NL_SET_ERR_MSG(info->extack, 1246 "No DMA-capable device found for netmem TX"); 1247 goto err_unlock_netdev; 1248 } 1249 1250 if (bind_dev != netdev) 1251 netdev_lock(bind_dev); 1252 1253 dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX); 1254 1255 binding = net_devmem_bind_dmabuf(bind_dev, 1256 bind_dev != netdev ? netdev : NULL, 1257 dma_dev, DMA_TO_DEVICE, dmabuf_fd, 1258 PAGE_SHIFT, priv, info->extack); 1259 if (IS_ERR(binding)) { 1260 err = PTR_ERR(binding); 1261 goto err_unlock_bind_dev; 1262 } 1263 1264 /* rsp was allocated large enough */ 1265 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id)); 1266 1267 genlmsg_end(rsp, hdr); 1268 1269 if (bind_dev != netdev) 1270 netdev_unlock(bind_dev); 1271 netdev_unlock(netdev); 1272 mutex_unlock(&priv->lock); 1273 1274 return genlmsg_reply(rsp, info); 1275 1276 err_unlock_bind_dev: 1277 if (bind_dev != netdev) 1278 netdev_unlock(bind_dev); 1279 err_unlock_netdev: 1280 netdev_unlock(netdev); 1281 err_unlock_sock: 1282 mutex_unlock(&priv->lock); 1283 err_genlmsg_free: 1284 nlmsg_free(rsp); 1285 return err; 1286 } 1287 1288 int netdev_nl_queue_create_doit(struct sk_buff *skb, struct genl_info *info) 1289 { 1290 const int qmaxtype = ARRAY_SIZE(netdev_queue_id_nl_policy) - 1; 1291 const int lmaxtype = ARRAY_SIZE(netdev_lease_nl_policy) - 1; 1292 int err, ifindex, ifindex_lease, queue_id, queue_id_lease; 1293 struct nlattr *qtb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 1294 struct nlattr *ltb[ARRAY_SIZE(netdev_lease_nl_policy)]; 1295 struct netdev_rx_queue *rxq, *rxq_lease; 1296 struct net_device *dev, *dev_lease; 1297 netdevice_tracker dev_tracker; 1298 s32 netns_lease = -1; 1299 struct nlattr *nest; 1300 struct sk_buff *rsp; 1301 struct net *net; 1302 void *hdr; 1303 1304 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX) || 1305 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 1306 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_LEASE)) 1307 return -EINVAL; 1308 if (nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]) != 1309 NETDEV_QUEUE_TYPE_RX) { 1310 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_QUEUE_TYPE]); 1311 return -EINVAL; 1312 } 1313 1314 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 1315 1316 nest = info->attrs[NETDEV_A_QUEUE_LEASE]; 1317 err = nla_parse_nested(ltb, lmaxtype, nest, 1318 netdev_lease_nl_policy, info->extack); 1319 if (err < 0) 1320 return err; 1321 if (NL_REQ_ATTR_CHECK(info->extack, nest, ltb, NETDEV_A_LEASE_IFINDEX) || 1322 NL_REQ_ATTR_CHECK(info->extack, nest, ltb, NETDEV_A_LEASE_QUEUE)) 1323 return -EINVAL; 1324 if (ltb[NETDEV_A_LEASE_NETNS_ID]) { 1325 if (!capable(CAP_NET_ADMIN)) 1326 return -EPERM; 1327 netns_lease = nla_get_s32(ltb[NETDEV_A_LEASE_NETNS_ID]); 1328 } 1329 1330 ifindex_lease = nla_get_u32(ltb[NETDEV_A_LEASE_IFINDEX]); 1331 1332 nest = ltb[NETDEV_A_LEASE_QUEUE]; 1333 err = nla_parse_nested(qtb, qmaxtype, nest, 1334 netdev_queue_id_nl_policy, info->extack); 1335 if (err < 0) 1336 return err; 1337 if (NL_REQ_ATTR_CHECK(info->extack, nest, qtb, NETDEV_A_QUEUE_ID) || 1338 NL_REQ_ATTR_CHECK(info->extack, nest, qtb, NETDEV_A_QUEUE_TYPE)) 1339 return -EINVAL; 1340 if (nla_get_u32(qtb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 1341 NL_SET_BAD_ATTR(info->extack, qtb[NETDEV_A_QUEUE_TYPE]); 1342 return -EINVAL; 1343 } 1344 1345 queue_id_lease = nla_get_u32(qtb[NETDEV_A_QUEUE_ID]); 1346 1347 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 1348 if (!rsp) 1349 return -ENOMEM; 1350 1351 hdr = genlmsg_iput(rsp, info); 1352 if (!hdr) { 1353 err = -EMSGSIZE; 1354 goto err_genlmsg_free; 1355 } 1356 1357 /* Locking order is always from the virtual to the physical device 1358 * since this is also the same order when applications open the 1359 * memory provider later on. 1360 */ 1361 dev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1362 if (!dev) { 1363 err = -ENODEV; 1364 goto err_genlmsg_free; 1365 } 1366 if (!netdev_can_create_queue(dev, info->extack)) { 1367 err = -EINVAL; 1368 goto err_unlock_dev; 1369 } 1370 1371 net = genl_info_net(info); 1372 if (netns_lease >= 0) { 1373 net = get_net_ns_by_id(net, netns_lease); 1374 if (!net) { 1375 err = -ENONET; 1376 goto err_unlock_dev; 1377 } 1378 } 1379 1380 dev_lease = netdev_get_by_index(net, ifindex_lease, &dev_tracker, 1381 GFP_KERNEL); 1382 if (!dev_lease) { 1383 err = -ENODEV; 1384 goto err_put_netns; 1385 } 1386 if (!netdev_can_lease_queue(dev_lease, info->extack)) { 1387 netdev_put(dev_lease, &dev_tracker); 1388 err = -EINVAL; 1389 goto err_put_netns; 1390 } 1391 1392 dev_lease = netdev_put_lock(dev_lease, net, &dev_tracker); 1393 if (!dev_lease) { 1394 err = -ENODEV; 1395 goto err_put_netns; 1396 } 1397 if (queue_id_lease >= dev_lease->real_num_rx_queues) { 1398 err = -ERANGE; 1399 NL_SET_BAD_ATTR(info->extack, qtb[NETDEV_A_QUEUE_ID]); 1400 goto err_unlock_dev_lease; 1401 } 1402 if (netdev_queue_busy(dev_lease, queue_id_lease, NETDEV_QUEUE_TYPE_RX, 1403 info->extack)) { 1404 err = -EBUSY; 1405 goto err_unlock_dev_lease; 1406 } 1407 1408 rxq_lease = __netif_get_rx_queue(dev_lease, queue_id_lease); 1409 rxq = __netif_get_rx_queue(dev, dev->real_num_rx_queues - 1); 1410 1411 /* Leasing queues from different physical devices is currently 1412 * not supported. Capabilities such as XDP features and DMA 1413 * device may differ between physical devices, and computing 1414 * a correct intersection for the virtual device is not yet 1415 * implemented. 1416 */ 1417 if (rxq->lease && rxq->lease->dev != dev_lease) { 1418 err = -EOPNOTSUPP; 1419 NL_SET_ERR_MSG(info->extack, 1420 "Leasing queues from different devices not supported"); 1421 goto err_unlock_dev_lease; 1422 } 1423 1424 queue_id = dev->queue_mgmt_ops->ndo_queue_create(dev, info->extack); 1425 if (queue_id < 0) { 1426 err = queue_id; 1427 goto err_unlock_dev_lease; 1428 } 1429 rxq = __netif_get_rx_queue(dev, queue_id); 1430 1431 netdev_rx_queue_lease(rxq, rxq_lease); 1432 1433 /* rsp was allocated large enough */ 1434 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_QUEUE_ID, queue_id)); 1435 1436 genlmsg_end(rsp, hdr); 1437 1438 netdev_unlock(dev_lease); 1439 netdev_unlock(dev); 1440 if (netns_lease >= 0) 1441 put_net(net); 1442 1443 return genlmsg_reply(rsp, info); 1444 1445 err_unlock_dev_lease: 1446 netdev_unlock(dev_lease); 1447 err_put_netns: 1448 if (netns_lease >= 0) 1449 put_net(net); 1450 err_unlock_dev: 1451 netdev_unlock(dev); 1452 err_genlmsg_free: 1453 nlmsg_free(rsp); 1454 return err; 1455 } 1456 1457 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv) 1458 { 1459 INIT_LIST_HEAD(&priv->bindings); 1460 mutex_init(&priv->lock); 1461 } 1462 1463 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv) 1464 { 1465 struct net_devmem_dmabuf_binding *binding; 1466 struct net_devmem_dmabuf_binding *temp; 1467 netdevice_tracker dev_tracker; 1468 struct net_device *dev; 1469 1470 mutex_lock(&priv->lock); 1471 list_for_each_entry_safe(binding, temp, &priv->bindings, list) { 1472 mutex_lock(&binding->lock); 1473 dev = binding->dev; 1474 if (!dev) { 1475 mutex_unlock(&binding->lock); 1476 net_devmem_unbind_dmabuf(binding); 1477 continue; 1478 } 1479 netdev_hold(dev, &dev_tracker, GFP_KERNEL); 1480 mutex_unlock(&binding->lock); 1481 1482 netdev_lock(dev); 1483 net_devmem_unbind_dmabuf(binding); 1484 netdev_unlock(dev); 1485 netdev_put(dev, &dev_tracker); 1486 } 1487 mutex_unlock(&priv->lock); 1488 } 1489 1490 static int netdev_genl_netdevice_event(struct notifier_block *nb, 1491 unsigned long event, void *ptr) 1492 { 1493 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 1494 1495 switch (event) { 1496 case NETDEV_REGISTER: 1497 netdev_lock_ops_to_full(netdev); 1498 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 1499 netdev_unlock_full_to_ops(netdev); 1500 break; 1501 case NETDEV_UNREGISTER: 1502 netdev_lock(netdev); 1503 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 1504 netdev_unlock(netdev); 1505 break; 1506 case NETDEV_XDP_FEAT_CHANGE: 1507 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 1508 break; 1509 } 1510 1511 return NOTIFY_OK; 1512 } 1513 1514 static struct notifier_block netdev_genl_nb = { 1515 .notifier_call = netdev_genl_netdevice_event, 1516 }; 1517 1518 static int __init netdev_genl_init(void) 1519 { 1520 int err; 1521 1522 err = register_netdevice_notifier(&netdev_genl_nb); 1523 if (err) 1524 return err; 1525 1526 err = genl_register_family(&netdev_nl_family); 1527 if (err) 1528 goto err_unreg_ntf; 1529 1530 return 0; 1531 1532 err_unreg_ntf: 1533 unregister_netdevice_notifier(&netdev_genl_nb); 1534 return err; 1535 } 1536 1537 subsys_initcall(netdev_genl_init); 1538