1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/pid_namespace.h> 6 #include <linux/rtnetlink.h> 7 #include <net/busy_poll.h> 8 #include <net/net_namespace.h> 9 #include <net/netdev_queues.h> 10 #include <net/netdev_rx_queue.h> 11 #include <net/sock.h> 12 #include <net/xdp.h> 13 #include <net/xdp_sock.h> 14 #include <net/page_pool/memory_provider.h> 15 16 #include "dev.h" 17 #include "devmem.h" 18 #include "netdev-genl-gen.h" 19 20 struct netdev_nl_dump_ctx { 21 unsigned long ifindex; 22 unsigned int rxq_idx; 23 unsigned int txq_idx; 24 unsigned int napi_id; 25 }; 26 27 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 28 { 29 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 30 31 return (struct netdev_nl_dump_ctx *)cb->ctx; 32 } 33 34 static int 35 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 36 const struct genl_info *info) 37 { 38 u64 xsk_features = 0; 39 u64 xdp_rx_meta = 0; 40 void *hdr; 41 42 netdev_assert_locked(netdev); /* note: rtnl_lock may not be held! */ 43 44 hdr = genlmsg_iput(rsp, info); 45 if (!hdr) 46 return -EMSGSIZE; 47 48 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 49 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 50 xdp_rx_meta |= flag; 51 XDP_METADATA_KFUNC_xxx 52 #undef XDP_METADATA_KFUNC 53 54 if (netdev->xsk_tx_metadata_ops) { 55 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 56 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 57 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 58 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 59 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time) 60 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO; 61 } 62 63 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 64 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 65 netdev->xdp_features, NETDEV_A_DEV_PAD) || 66 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 67 xdp_rx_meta, NETDEV_A_DEV_PAD) || 68 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 69 xsk_features, NETDEV_A_DEV_PAD)) 70 goto err_cancel_msg; 71 72 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 73 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 74 netdev->xdp_zc_max_segs)) 75 goto err_cancel_msg; 76 } 77 78 genlmsg_end(rsp, hdr); 79 80 return 0; 81 82 err_cancel_msg: 83 genlmsg_cancel(rsp, hdr); 84 return -EMSGSIZE; 85 } 86 87 static void 88 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 89 { 90 struct genl_info info; 91 struct sk_buff *ntf; 92 93 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 94 NETDEV_NLGRP_MGMT)) 95 return; 96 97 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 98 99 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 100 if (!ntf) 101 return; 102 103 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 104 nlmsg_free(ntf); 105 return; 106 } 107 108 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 109 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 110 } 111 112 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 113 { 114 struct net_device *netdev; 115 struct sk_buff *rsp; 116 u32 ifindex; 117 int err; 118 119 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 120 return -EINVAL; 121 122 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 123 124 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 125 if (!rsp) 126 return -ENOMEM; 127 128 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 129 if (!netdev) { 130 err = -ENODEV; 131 goto err_free_msg; 132 } 133 134 err = netdev_nl_dev_fill(netdev, rsp, info); 135 netdev_unlock(netdev); 136 137 if (err) 138 goto err_free_msg; 139 140 return genlmsg_reply(rsp, info); 141 142 err_free_msg: 143 nlmsg_free(rsp); 144 return err; 145 } 146 147 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 148 { 149 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 150 struct net *net = sock_net(skb->sk); 151 int err; 152 153 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 155 if (err < 0) 156 return err; 157 } 158 159 return 0; 160 } 161 162 static int 163 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 164 const struct genl_info *info) 165 { 166 unsigned long irq_suspend_timeout; 167 unsigned long gro_flush_timeout; 168 u32 napi_defer_hard_irqs; 169 void *hdr; 170 pid_t pid; 171 172 if (!napi->dev->up) 173 return 0; 174 175 hdr = genlmsg_iput(rsp, info); 176 if (!hdr) 177 return -EMSGSIZE; 178 179 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 180 goto nla_put_failure; 181 182 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 183 goto nla_put_failure; 184 185 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 186 goto nla_put_failure; 187 188 if (nla_put_uint(rsp, NETDEV_A_NAPI_THREADED, 189 napi_get_threaded(napi))) 190 goto nla_put_failure; 191 192 if (napi->thread) { 193 pid = task_pid_nr_ns(napi->thread, 194 task_active_pid_ns(current)); 195 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 196 goto nla_put_failure; 197 } 198 199 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 200 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 201 napi_defer_hard_irqs)) 202 goto nla_put_failure; 203 204 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 205 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 206 irq_suspend_timeout)) 207 goto nla_put_failure; 208 209 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 210 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 211 gro_flush_timeout)) 212 goto nla_put_failure; 213 214 genlmsg_end(rsp, hdr); 215 216 return 0; 217 218 nla_put_failure: 219 genlmsg_cancel(rsp, hdr); 220 return -EMSGSIZE; 221 } 222 223 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 224 { 225 struct napi_struct *napi; 226 struct sk_buff *rsp; 227 u32 napi_id; 228 int err; 229 230 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 231 return -EINVAL; 232 233 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 234 235 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 236 if (!rsp) 237 return -ENOMEM; 238 239 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 240 if (napi) { 241 err = netdev_nl_napi_fill_one(rsp, napi, info); 242 netdev_unlock(napi->dev); 243 } else { 244 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 245 err = -ENOENT; 246 } 247 248 if (err) { 249 goto err_free_msg; 250 } else if (!rsp->len) { 251 err = -ENOENT; 252 goto err_free_msg; 253 } 254 255 return genlmsg_reply(rsp, info); 256 257 err_free_msg: 258 nlmsg_free(rsp); 259 return err; 260 } 261 262 static int 263 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 264 const struct genl_info *info, 265 struct netdev_nl_dump_ctx *ctx) 266 { 267 struct napi_struct *napi; 268 unsigned int prev_id; 269 int err = 0; 270 271 if (!netdev->up) 272 return err; 273 274 prev_id = UINT_MAX; 275 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 276 if (!napi_id_valid(napi->napi_id)) 277 continue; 278 279 /* Dump continuation below depends on the list being sorted */ 280 WARN_ON_ONCE(napi->napi_id >= prev_id); 281 prev_id = napi->napi_id; 282 283 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 284 continue; 285 286 err = netdev_nl_napi_fill_one(rsp, napi, info); 287 if (err) 288 return err; 289 ctx->napi_id = napi->napi_id; 290 } 291 return err; 292 } 293 294 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 295 { 296 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 297 const struct genl_info *info = genl_info_dump(cb); 298 struct net *net = sock_net(skb->sk); 299 struct net_device *netdev; 300 u32 ifindex = 0; 301 int err = 0; 302 303 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 304 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 305 306 if (ifindex) { 307 netdev = netdev_get_by_index_lock(net, ifindex); 308 if (netdev) { 309 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 310 netdev_unlock(netdev); 311 } else { 312 err = -ENODEV; 313 } 314 } else { 315 unsigned long start_ifindex = ctx->ifindex; 316 317 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 318 if (ctx->ifindex != start_ifindex) 319 ctx->napi_id = 0; 320 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 321 if (err < 0) 322 break; 323 } 324 } 325 326 return err; 327 } 328 329 static int 330 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 331 { 332 u64 irq_suspend_timeout = 0; 333 u64 gro_flush_timeout = 0; 334 u8 threaded = 0; 335 u32 defer = 0; 336 337 if (info->attrs[NETDEV_A_NAPI_THREADED]) { 338 int ret; 339 340 threaded = nla_get_uint(info->attrs[NETDEV_A_NAPI_THREADED]); 341 if (threaded && test_bit(NAPI_STATE_PERCPU, &napi->state)) { 342 NL_SET_BAD_ATTR(info->extack, 343 info->attrs[NETDEV_A_NAPI_THREADED]); 344 return -EOPNOTSUPP; 345 } 346 347 ret = napi_set_threaded(napi, threaded); 348 if (ret) 349 return ret; 350 } 351 352 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 353 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 354 napi_set_defer_hard_irqs(napi, defer); 355 } 356 357 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 358 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 359 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 360 } 361 362 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 363 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 364 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 365 } 366 367 return 0; 368 } 369 370 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 371 { 372 struct napi_struct *napi; 373 unsigned int napi_id; 374 int err; 375 376 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 377 return -EINVAL; 378 379 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 380 381 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 382 if (napi) { 383 err = netdev_nl_napi_set_config(napi, info); 384 netdev_unlock(napi->dev); 385 } else { 386 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 387 err = -ENOENT; 388 } 389 390 return err; 391 } 392 393 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi) 394 { 395 if (napi && napi_id_valid(napi->napi_id)) 396 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id); 397 return 0; 398 } 399 400 static int 401 netdev_nl_queue_fill_lease(struct sk_buff *rsp, struct net_device *netdev, 402 u32 q_idx, u32 q_type) 403 { 404 struct net_device *orig_netdev = netdev; 405 struct nlattr *nest_lease, *nest_queue; 406 struct netdev_rx_queue *rxq; 407 struct net *net, *peer_net; 408 409 rxq = __netif_get_rx_queue_lease(&netdev, &q_idx, NETIF_PHYS_TO_VIRT); 410 if (!rxq || orig_netdev == netdev) 411 return 0; 412 413 nest_lease = nla_nest_start(rsp, NETDEV_A_QUEUE_LEASE); 414 if (!nest_lease) 415 goto nla_put_failure; 416 417 nest_queue = nla_nest_start(rsp, NETDEV_A_LEASE_QUEUE); 418 if (!nest_queue) 419 goto nla_put_failure; 420 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx)) 421 goto nla_put_failure; 422 if (nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type)) 423 goto nla_put_failure; 424 nla_nest_end(rsp, nest_queue); 425 426 if (nla_put_u32(rsp, NETDEV_A_LEASE_IFINDEX, 427 READ_ONCE(netdev->ifindex))) 428 goto nla_put_failure; 429 430 rcu_read_lock(); 431 peer_net = dev_net_rcu(netdev); 432 net = dev_net_rcu(orig_netdev); 433 if (!net_eq(net, peer_net)) { 434 s32 id = peernet2id_alloc(net, peer_net, GFP_ATOMIC); 435 436 if (nla_put_s32(rsp, NETDEV_A_LEASE_NETNS_ID, id)) 437 goto nla_put_failure_unlock; 438 } 439 rcu_read_unlock(); 440 nla_nest_end(rsp, nest_lease); 441 return 0; 442 443 nla_put_failure_unlock: 444 rcu_read_unlock(); 445 nla_put_failure: 446 return -EMSGSIZE; 447 } 448 449 static int 450 __netdev_nl_queue_fill_mp(struct sk_buff *rsp, struct netdev_rx_queue *rxq) 451 { 452 struct pp_memory_provider_params *params = &rxq->mp_params; 453 454 if (params->mp_ops && 455 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq)) 456 return -EMSGSIZE; 457 458 #ifdef CONFIG_XDP_SOCKETS 459 if (rxq->pool) 460 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 461 return -EMSGSIZE; 462 #endif 463 return 0; 464 } 465 466 static int 467 netdev_nl_queue_fill_mp(struct sk_buff *rsp, struct net_device *netdev, 468 struct netdev_rx_queue *rxq) 469 { 470 struct netdev_rx_queue *hw_rxq; 471 int ret; 472 473 hw_rxq = rxq->lease; 474 if (!hw_rxq || !netif_is_queue_leasee(netdev)) 475 return __netdev_nl_queue_fill_mp(rsp, rxq); 476 477 netdev_lock(hw_rxq->dev); 478 ret = __netdev_nl_queue_fill_mp(rsp, hw_rxq); 479 netdev_unlock(hw_rxq->dev); 480 return ret; 481 } 482 483 static int 484 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 485 u32 q_idx, u32 q_type, const struct genl_info *info) 486 { 487 struct netdev_rx_queue *rxq; 488 struct netdev_queue *txq; 489 void *hdr; 490 491 hdr = genlmsg_iput(rsp, info); 492 if (!hdr) 493 return -EMSGSIZE; 494 495 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 496 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 497 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 498 goto nla_put_failure; 499 500 switch (q_type) { 501 case NETDEV_QUEUE_TYPE_RX: 502 rxq = __netif_get_rx_queue(netdev, q_idx); 503 if (nla_put_napi_id(rsp, rxq->napi)) 504 goto nla_put_failure; 505 if (netdev_nl_queue_fill_lease(rsp, netdev, q_idx, q_type)) 506 goto nla_put_failure; 507 if (netdev_nl_queue_fill_mp(rsp, netdev, rxq)) 508 goto nla_put_failure; 509 break; 510 case NETDEV_QUEUE_TYPE_TX: 511 txq = netdev_get_tx_queue(netdev, q_idx); 512 if (nla_put_napi_id(rsp, txq->napi)) 513 goto nla_put_failure; 514 #ifdef CONFIG_XDP_SOCKETS 515 if (txq->pool) 516 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 517 goto nla_put_failure; 518 #endif 519 break; 520 } 521 522 genlmsg_end(rsp, hdr); 523 524 return 0; 525 526 nla_put_failure: 527 genlmsg_cancel(rsp, hdr); 528 return -EMSGSIZE; 529 } 530 531 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 532 u32 q_type) 533 { 534 switch (q_type) { 535 case NETDEV_QUEUE_TYPE_RX: 536 if (q_id >= netdev->real_num_rx_queues) 537 return -EINVAL; 538 return 0; 539 case NETDEV_QUEUE_TYPE_TX: 540 if (q_id >= netdev->real_num_tx_queues) 541 return -EINVAL; 542 } 543 return 0; 544 } 545 546 static int 547 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 548 u32 q_type, const struct genl_info *info) 549 { 550 int err; 551 552 if (!netdev->up) 553 return -ENOENT; 554 555 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 556 if (err) 557 return err; 558 559 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 560 } 561 562 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 563 { 564 u32 q_id, q_type, ifindex; 565 struct net_device *netdev; 566 struct sk_buff *rsp; 567 int err; 568 569 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 570 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 571 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 572 return -EINVAL; 573 574 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 575 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 576 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 577 578 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 579 if (!rsp) 580 return -ENOMEM; 581 582 netdev = netdev_get_by_index_lock_ops_compat(genl_info_net(info), 583 ifindex); 584 if (netdev) { 585 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 586 netdev_unlock_ops_compat(netdev); 587 } else { 588 err = -ENODEV; 589 } 590 591 if (err) 592 goto err_free_msg; 593 594 return genlmsg_reply(rsp, info); 595 596 err_free_msg: 597 nlmsg_free(rsp); 598 return err; 599 } 600 601 static int 602 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 603 const struct genl_info *info, 604 struct netdev_nl_dump_ctx *ctx) 605 { 606 int err = 0; 607 608 if (!netdev->up) 609 return err; 610 611 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) { 612 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx, 613 NETDEV_QUEUE_TYPE_RX, info); 614 if (err) 615 return err; 616 } 617 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) { 618 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx, 619 NETDEV_QUEUE_TYPE_TX, info); 620 if (err) 621 return err; 622 } 623 624 return err; 625 } 626 627 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 628 { 629 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 630 const struct genl_info *info = genl_info_dump(cb); 631 struct net *net = sock_net(skb->sk); 632 struct net_device *netdev; 633 u32 ifindex = 0; 634 int err = 0; 635 636 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 637 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 638 639 if (ifindex) { 640 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex); 641 if (netdev) { 642 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 643 netdev_unlock_ops_compat(netdev); 644 } else { 645 err = -ENODEV; 646 } 647 } else { 648 unsigned long start_ifindex = ctx->ifindex; 649 650 for_each_netdev_lock_ops_compat_scoped(net, netdev, 651 ctx->ifindex) { 652 if (ctx->ifindex != start_ifindex) { 653 ctx->rxq_idx = 0; 654 ctx->txq_idx = 0; 655 } 656 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 657 if (err < 0) 658 break; 659 } 660 } 661 662 return err; 663 } 664 665 #define NETDEV_STAT_NOT_SET (~0ULL) 666 667 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 668 { 669 const u64 *add = _add; 670 u64 *sum = _sum; 671 672 while (size) { 673 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 674 *sum += *add; 675 sum++; 676 add++; 677 size -= 8; 678 } 679 } 680 681 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 682 { 683 if (value == NETDEV_STAT_NOT_SET) 684 return 0; 685 return nla_put_uint(rsp, attr_id, value); 686 } 687 688 static int 689 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 690 { 691 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 692 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 693 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 694 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 695 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 696 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) || 697 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 698 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 699 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 700 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 701 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 702 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 703 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 704 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 705 return -EMSGSIZE; 706 return 0; 707 } 708 709 static int 710 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 711 { 712 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 713 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 714 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 715 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 716 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 717 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 718 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 719 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 720 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 721 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 722 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 723 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 724 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 725 return -EMSGSIZE; 726 return 0; 727 } 728 729 static int 730 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 731 u32 q_type, int i, const struct genl_info *info) 732 { 733 const struct netdev_stat_ops *ops = netdev->stat_ops; 734 struct netdev_queue_stats_rx rx; 735 struct netdev_queue_stats_tx tx; 736 void *hdr; 737 738 hdr = genlmsg_iput(rsp, info); 739 if (!hdr) 740 return -EMSGSIZE; 741 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 742 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 743 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 744 goto nla_put_failure; 745 746 switch (q_type) { 747 case NETDEV_QUEUE_TYPE_RX: 748 memset(&rx, 0xff, sizeof(rx)); 749 ops->get_queue_stats_rx(netdev, i, &rx); 750 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 751 goto nla_cancel; 752 if (netdev_nl_stats_write_rx(rsp, &rx)) 753 goto nla_put_failure; 754 break; 755 case NETDEV_QUEUE_TYPE_TX: 756 memset(&tx, 0xff, sizeof(tx)); 757 ops->get_queue_stats_tx(netdev, i, &tx); 758 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 759 goto nla_cancel; 760 if (netdev_nl_stats_write_tx(rsp, &tx)) 761 goto nla_put_failure; 762 break; 763 } 764 765 genlmsg_end(rsp, hdr); 766 return 0; 767 768 nla_cancel: 769 genlmsg_cancel(rsp, hdr); 770 return 0; 771 nla_put_failure: 772 genlmsg_cancel(rsp, hdr); 773 return -EMSGSIZE; 774 } 775 776 static int 777 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 778 const struct genl_info *info, 779 struct netdev_nl_dump_ctx *ctx) 780 { 781 const struct netdev_stat_ops *ops = netdev->stat_ops; 782 int i, err; 783 784 if (!netdev->up) 785 return 0; 786 787 i = ctx->rxq_idx; 788 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 789 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 790 i, info); 791 if (err) 792 return err; 793 ctx->rxq_idx = ++i; 794 } 795 i = ctx->txq_idx; 796 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 797 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 798 i, info); 799 if (err) 800 return err; 801 ctx->txq_idx = ++i; 802 } 803 804 return 0; 805 } 806 807 /** 808 * netdev_stat_queue_sum() - add up queue stats from range of queues 809 * @netdev: net_device 810 * @rx_start: index of the first Rx queue to query 811 * @rx_end: index after the last Rx queue (first *not* to query) 812 * @rx_sum: output Rx stats, should be already initialized 813 * @tx_start: index of the first Tx queue to query 814 * @tx_end: index after the last Tx queue (first *not* to query) 815 * @tx_sum: output Tx stats, should be already initialized 816 * 817 * Add stats from [start, end) range of queue IDs to *x_sum structs. 818 * The sum structs must be already initialized. Usually this 819 * helper is invoked from the .get_base_stats callbacks of drivers 820 * to account for stats of disabled queues. In that case the ranges 821 * are usually [netdev->real_num_*x_queues, netdev->num_*x_queues). 822 */ 823 void netdev_stat_queue_sum(struct net_device *netdev, 824 int rx_start, int rx_end, 825 struct netdev_queue_stats_rx *rx_sum, 826 int tx_start, int tx_end, 827 struct netdev_queue_stats_tx *tx_sum) 828 { 829 const struct netdev_stat_ops *ops; 830 struct netdev_queue_stats_rx rx; 831 struct netdev_queue_stats_tx tx; 832 int i; 833 834 ops = netdev->stat_ops; 835 836 for (i = rx_start; i < rx_end; i++) { 837 memset(&rx, 0xff, sizeof(rx)); 838 if (ops->get_queue_stats_rx) 839 ops->get_queue_stats_rx(netdev, i, &rx); 840 netdev_nl_stats_add(rx_sum, &rx, sizeof(rx)); 841 } 842 for (i = tx_start; i < tx_end; i++) { 843 memset(&tx, 0xff, sizeof(tx)); 844 if (ops->get_queue_stats_tx) 845 ops->get_queue_stats_tx(netdev, i, &tx); 846 netdev_nl_stats_add(tx_sum, &tx, sizeof(tx)); 847 } 848 } 849 EXPORT_SYMBOL(netdev_stat_queue_sum); 850 851 static int 852 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 853 const struct genl_info *info) 854 { 855 struct netdev_queue_stats_rx rx_sum; 856 struct netdev_queue_stats_tx tx_sum; 857 void *hdr; 858 859 /* Netdev can't guarantee any complete counters */ 860 if (!netdev->stat_ops->get_base_stats) 861 return 0; 862 863 memset(&rx_sum, 0xff, sizeof(rx_sum)); 864 memset(&tx_sum, 0xff, sizeof(tx_sum)); 865 866 netdev->stat_ops->get_base_stats(netdev, &rx_sum, &tx_sum); 867 868 /* The op was there, but nothing reported, don't bother */ 869 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 870 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 871 return 0; 872 873 hdr = genlmsg_iput(rsp, info); 874 if (!hdr) 875 return -EMSGSIZE; 876 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 877 goto nla_put_failure; 878 879 netdev_stat_queue_sum(netdev, 0, netdev->real_num_rx_queues, &rx_sum, 880 0, netdev->real_num_tx_queues, &tx_sum); 881 882 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 883 netdev_nl_stats_write_tx(rsp, &tx_sum)) 884 goto nla_put_failure; 885 886 genlmsg_end(rsp, hdr); 887 return 0; 888 889 nla_put_failure: 890 genlmsg_cancel(rsp, hdr); 891 return -EMSGSIZE; 892 } 893 894 static int 895 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 896 struct sk_buff *skb, const struct genl_info *info, 897 struct netdev_nl_dump_ctx *ctx) 898 { 899 if (!netdev->stat_ops) 900 return 0; 901 902 switch (scope) { 903 case 0: 904 return netdev_nl_stats_by_netdev(netdev, skb, info); 905 case NETDEV_QSTATS_SCOPE_QUEUE: 906 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 907 } 908 909 return -EINVAL; /* Should not happen, per netlink policy */ 910 } 911 912 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 913 struct netlink_callback *cb) 914 { 915 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 916 const struct genl_info *info = genl_info_dump(cb); 917 struct net *net = sock_net(skb->sk); 918 unsigned long start_ifindex; 919 struct net_device *netdev; 920 unsigned int ifindex; 921 unsigned int scope; 922 int err = 0; 923 924 scope = 0; 925 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 926 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 927 928 ifindex = 0; 929 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 930 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 931 932 if (ifindex) { 933 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex); 934 if (!netdev) { 935 NL_SET_BAD_ATTR(info->extack, 936 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 937 return -ENODEV; 938 } 939 if (netdev->stat_ops) { 940 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 941 info, ctx); 942 } else { 943 NL_SET_BAD_ATTR(info->extack, 944 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 945 err = -EOPNOTSUPP; 946 } 947 netdev_unlock_ops_compat(netdev); 948 return err; 949 } 950 951 start_ifindex = ctx->ifindex; 952 953 for_each_netdev_lock_ops_compat_scoped(net, netdev, ctx->ifindex) { 954 if (ctx->ifindex != start_ifindex) { 955 ctx->rxq_idx = 0; 956 ctx->txq_idx = 0; 957 } 958 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 959 info, ctx); 960 if (err < 0) 961 break; 962 } 963 964 return err; 965 } 966 967 static int netdev_nl_read_rxq_bitmap(struct genl_info *info, 968 u32 rxq_bitmap_len, 969 unsigned long *rxq_bitmap) 970 { 971 const int maxtype = ARRAY_SIZE(netdev_queue_id_nl_policy) - 1; 972 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 973 struct nlattr *attr; 974 int rem, err = 0; 975 u32 rxq_idx; 976 977 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 978 genlmsg_data(info->genlhdr), 979 genlmsg_len(info->genlhdr), rem) { 980 err = nla_parse_nested(tb, maxtype, attr, 981 netdev_queue_id_nl_policy, info->extack); 982 if (err < 0) 983 return err; 984 985 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 986 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) 987 return -EINVAL; 988 989 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 990 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 991 return -EINVAL; 992 } 993 994 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 995 if (rxq_idx >= rxq_bitmap_len) { 996 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_ID]); 997 return -EINVAL; 998 } 999 1000 bitmap_set(rxq_bitmap, rxq_idx, 1); 1001 } 1002 1003 return 0; 1004 } 1005 1006 static struct device * 1007 netdev_nl_get_dma_dev(struct net_device *netdev, unsigned long *rxq_bitmap, 1008 struct netlink_ext_ack *extack) 1009 { 1010 struct device *dma_dev = NULL; 1011 u32 rxq_idx, prev_rxq_idx; 1012 1013 for_each_set_bit(rxq_idx, rxq_bitmap, netdev->real_num_rx_queues) { 1014 struct device *rxq_dma_dev; 1015 1016 rxq_dma_dev = netdev_queue_get_dma_dev(netdev, rxq_idx, 1017 NETDEV_QUEUE_TYPE_RX); 1018 if (dma_dev && rxq_dma_dev != dma_dev) { 1019 NL_SET_ERR_MSG_FMT(extack, "DMA device mismatch between queue %u and %u (multi-PF device?)", 1020 rxq_idx, prev_rxq_idx); 1021 return ERR_PTR(-EOPNOTSUPP); 1022 } 1023 1024 dma_dev = rxq_dma_dev; 1025 prev_rxq_idx = rxq_idx; 1026 } 1027 1028 return dma_dev; 1029 } 1030 1031 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 1032 { 1033 struct net_devmem_dmabuf_binding *binding; 1034 unsigned int niov_shift = PAGE_SHIFT; 1035 u32 ifindex, dmabuf_fd, rxq_idx; 1036 struct netdev_nl_sock *priv; 1037 struct net_device *netdev; 1038 unsigned long *rxq_bitmap; 1039 struct device *dma_dev; 1040 struct sk_buff *rsp; 1041 int err = 0; 1042 void *hdr; 1043 1044 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 1045 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 1046 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 1047 return -EINVAL; 1048 1049 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 1050 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 1051 1052 if (info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]) { 1053 u32 rx_page_size = nla_get_u32(info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE]); 1054 1055 if (!is_power_of_2(rx_page_size)) { 1056 NL_SET_ERR_MSG_ATTR(info->extack, 1057 info->attrs[NETDEV_A_DMABUF_RX_PAGE_SIZE], 1058 "rx-page-size must be a power of 2"); 1059 return -EINVAL; 1060 } 1061 niov_shift = ilog2(rx_page_size); 1062 } 1063 1064 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 1065 if (IS_ERR(priv)) 1066 return PTR_ERR(priv); 1067 1068 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 1069 if (!rsp) 1070 return -ENOMEM; 1071 1072 hdr = genlmsg_iput(rsp, info); 1073 if (!hdr) { 1074 err = -EMSGSIZE; 1075 goto err_genlmsg_free; 1076 } 1077 1078 mutex_lock(&priv->lock); 1079 1080 err = 0; 1081 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1082 if (!netdev) { 1083 err = -ENODEV; 1084 goto err_unlock_sock; 1085 } 1086 if (!netif_device_present(netdev)) 1087 err = -ENODEV; 1088 else if (!netdev_need_ops_lock(netdev)) 1089 err = -EOPNOTSUPP; 1090 if (err) { 1091 NL_SET_BAD_ATTR(info->extack, 1092 info->attrs[NETDEV_A_DEV_IFINDEX]); 1093 goto err_unlock; 1094 } 1095 1096 rxq_bitmap = bitmap_zalloc(netdev->real_num_rx_queues, GFP_KERNEL); 1097 if (!rxq_bitmap) { 1098 err = -ENOMEM; 1099 goto err_unlock; 1100 } 1101 1102 err = netdev_nl_read_rxq_bitmap(info, netdev->real_num_rx_queues, 1103 rxq_bitmap); 1104 if (err) 1105 goto err_rxq_bitmap; 1106 1107 dma_dev = netdev_nl_get_dma_dev(netdev, rxq_bitmap, info->extack); 1108 if (IS_ERR(dma_dev)) { 1109 err = PTR_ERR(dma_dev); 1110 goto err_rxq_bitmap; 1111 } 1112 1113 binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE, 1114 dmabuf_fd, niov_shift, priv, 1115 NETLINK_CB(skb).sk, info->extack); 1116 if (IS_ERR(binding)) { 1117 err = PTR_ERR(binding); 1118 goto err_rxq_bitmap; 1119 } 1120 1121 for_each_set_bit(rxq_idx, rxq_bitmap, netdev->real_num_rx_queues) { 1122 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 1123 info->extack); 1124 if (err) 1125 goto err_unbind; 1126 } 1127 1128 /* rsp was allocated large enough */ 1129 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id)); 1130 1131 genlmsg_end(rsp, hdr); 1132 1133 err = genlmsg_reply(rsp, info); 1134 1135 bitmap_free(rxq_bitmap); 1136 1137 netdev_unlock(netdev); 1138 1139 mutex_unlock(&priv->lock); 1140 1141 return err < 0 ? err : 0; 1142 1143 err_unbind: 1144 net_devmem_unbind_dmabuf(binding); 1145 err_rxq_bitmap: 1146 bitmap_free(rxq_bitmap); 1147 err_unlock: 1148 netdev_unlock(netdev); 1149 err_unlock_sock: 1150 mutex_unlock(&priv->lock); 1151 err_genlmsg_free: 1152 nlmsg_free(rsp); 1153 return err; 1154 } 1155 1156 /* Find the DMA-capable device for a netmem TX binding. 1157 * 1158 * For NETMEM_TX_DMA devices, return the device itself. 1159 * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying 1160 * physical device and return it. 1161 */ 1162 static struct net_device * 1163 netdev_find_netmem_tx_dev(struct net_device *dev) 1164 { 1165 struct netdev_rx_queue *lease_rxq; 1166 struct net_device *phys_dev; 1167 int i; 1168 1169 if (dev->netmem_tx == NETMEM_TX_DMA) 1170 return dev; 1171 1172 if (dev->netmem_tx != NETMEM_TX_NO_DMA) 1173 return NULL; 1174 1175 for (i = 0; i < dev->real_num_rx_queues; i++) { 1176 lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease); 1177 if (!lease_rxq) 1178 continue; 1179 1180 phys_dev = lease_rxq->dev; 1181 if (netif_device_present(phys_dev) && 1182 phys_dev->netmem_tx == NETMEM_TX_DMA) 1183 return phys_dev; 1184 } 1185 1186 return NULL; 1187 } 1188 1189 /* Note: NETDEV_CMD_BIND_TX is intentionally unprivileged (no 1190 * GENL_ADMIN_PERM / GENL_UNS_ADMIN_PERM). Unlike bind-rx, which configures 1191 * shared NIC RX queues, bind-tx only DMA-maps the caller's dmabuf so they can 1192 * transmit from it on their own sockets without affecting other traffic or 1193 * device state. 1194 */ 1195 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) 1196 { 1197 struct net_devmem_dmabuf_binding *binding; 1198 struct net_device *bind_dev; 1199 struct netdev_nl_sock *priv; 1200 struct net_device *netdev; 1201 struct device *dma_dev; 1202 u32 ifindex, dmabuf_fd; 1203 struct sk_buff *rsp; 1204 int err = 0; 1205 void *hdr; 1206 1207 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 1208 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD)) 1209 return -EINVAL; 1210 1211 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 1212 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 1213 1214 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 1215 if (IS_ERR(priv)) 1216 return PTR_ERR(priv); 1217 1218 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 1219 if (!rsp) 1220 return -ENOMEM; 1221 1222 hdr = genlmsg_iput(rsp, info); 1223 if (!hdr) { 1224 err = -EMSGSIZE; 1225 goto err_genlmsg_free; 1226 } 1227 1228 mutex_lock(&priv->lock); 1229 1230 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1231 if (!netdev) { 1232 err = -ENODEV; 1233 goto err_unlock_sock; 1234 } 1235 1236 if (!netif_device_present(netdev)) { 1237 err = -ENODEV; 1238 goto err_unlock_netdev; 1239 } 1240 1241 if (netdev->netmem_tx == NETMEM_TX_NONE) { 1242 err = -EOPNOTSUPP; 1243 NL_SET_ERR_MSG(info->extack, 1244 "Driver does not support netmem TX"); 1245 goto err_unlock_netdev; 1246 } 1247 1248 bind_dev = netdev_find_netmem_tx_dev(netdev); 1249 if (!bind_dev) { 1250 err = -EOPNOTSUPP; 1251 NL_SET_ERR_MSG(info->extack, 1252 "No DMA-capable device found for netmem TX"); 1253 goto err_unlock_netdev; 1254 } 1255 1256 if (bind_dev != netdev) 1257 netdev_lock(bind_dev); 1258 1259 dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX); 1260 1261 binding = net_devmem_bind_dmabuf(bind_dev, 1262 bind_dev != netdev ? netdev : NULL, 1263 dma_dev, DMA_TO_DEVICE, dmabuf_fd, 1264 PAGE_SHIFT, priv, NETLINK_CB(skb).sk, 1265 info->extack); 1266 if (IS_ERR(binding)) { 1267 err = PTR_ERR(binding); 1268 goto err_unlock_bind_dev; 1269 } 1270 1271 /* rsp was allocated large enough */ 1272 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id)); 1273 1274 genlmsg_end(rsp, hdr); 1275 1276 if (bind_dev != netdev) 1277 netdev_unlock(bind_dev); 1278 netdev_unlock(netdev); 1279 mutex_unlock(&priv->lock); 1280 1281 return genlmsg_reply(rsp, info); 1282 1283 err_unlock_bind_dev: 1284 if (bind_dev != netdev) 1285 netdev_unlock(bind_dev); 1286 err_unlock_netdev: 1287 netdev_unlock(netdev); 1288 err_unlock_sock: 1289 mutex_unlock(&priv->lock); 1290 err_genlmsg_free: 1291 nlmsg_free(rsp); 1292 return err; 1293 } 1294 1295 int netdev_nl_queue_create_doit(struct sk_buff *skb, struct genl_info *info) 1296 { 1297 const int qmaxtype = ARRAY_SIZE(netdev_queue_id_nl_policy) - 1; 1298 const int lmaxtype = ARRAY_SIZE(netdev_lease_nl_policy) - 1; 1299 int err, ifindex, ifindex_lease, queue_id, queue_id_lease; 1300 struct nlattr *qtb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 1301 struct nlattr *ltb[ARRAY_SIZE(netdev_lease_nl_policy)]; 1302 struct netdev_rx_queue *rxq, *rxq_lease; 1303 struct net_device *dev, *dev_lease; 1304 netdevice_tracker dev_tracker; 1305 s32 netns_lease = -1; 1306 struct nlattr *nest; 1307 struct sk_buff *rsp; 1308 struct net *net; 1309 void *hdr; 1310 1311 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX) || 1312 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 1313 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_LEASE)) 1314 return -EINVAL; 1315 if (nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]) != 1316 NETDEV_QUEUE_TYPE_RX) { 1317 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_QUEUE_TYPE]); 1318 return -EINVAL; 1319 } 1320 1321 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 1322 1323 nest = info->attrs[NETDEV_A_QUEUE_LEASE]; 1324 err = nla_parse_nested(ltb, lmaxtype, nest, 1325 netdev_lease_nl_policy, info->extack); 1326 if (err < 0) 1327 return err; 1328 if (NL_REQ_ATTR_CHECK(info->extack, nest, ltb, NETDEV_A_LEASE_IFINDEX) || 1329 NL_REQ_ATTR_CHECK(info->extack, nest, ltb, NETDEV_A_LEASE_QUEUE)) 1330 return -EINVAL; 1331 if (ltb[NETDEV_A_LEASE_NETNS_ID]) { 1332 if (!capable(CAP_NET_ADMIN)) 1333 return -EPERM; 1334 netns_lease = nla_get_s32(ltb[NETDEV_A_LEASE_NETNS_ID]); 1335 } 1336 1337 ifindex_lease = nla_get_u32(ltb[NETDEV_A_LEASE_IFINDEX]); 1338 1339 nest = ltb[NETDEV_A_LEASE_QUEUE]; 1340 err = nla_parse_nested(qtb, qmaxtype, nest, 1341 netdev_queue_id_nl_policy, info->extack); 1342 if (err < 0) 1343 return err; 1344 if (NL_REQ_ATTR_CHECK(info->extack, nest, qtb, NETDEV_A_QUEUE_ID) || 1345 NL_REQ_ATTR_CHECK(info->extack, nest, qtb, NETDEV_A_QUEUE_TYPE)) 1346 return -EINVAL; 1347 if (nla_get_u32(qtb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 1348 NL_SET_BAD_ATTR(info->extack, qtb[NETDEV_A_QUEUE_TYPE]); 1349 return -EINVAL; 1350 } 1351 1352 queue_id_lease = nla_get_u32(qtb[NETDEV_A_QUEUE_ID]); 1353 1354 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 1355 if (!rsp) 1356 return -ENOMEM; 1357 1358 hdr = genlmsg_iput(rsp, info); 1359 if (!hdr) { 1360 err = -EMSGSIZE; 1361 goto err_genlmsg_free; 1362 } 1363 1364 /* Locking order is always from the virtual to the physical device 1365 * since this is also the same order when applications open the 1366 * memory provider later on. 1367 */ 1368 dev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1369 if (!dev) { 1370 err = -ENODEV; 1371 goto err_genlmsg_free; 1372 } 1373 if (!netdev_can_create_queue(dev, info->extack)) { 1374 err = -EINVAL; 1375 goto err_unlock_dev; 1376 } 1377 1378 net = genl_info_net(info); 1379 if (netns_lease >= 0) { 1380 net = get_net_ns_by_id(net, netns_lease); 1381 if (!net) { 1382 err = -ENONET; 1383 goto err_unlock_dev; 1384 } 1385 } 1386 1387 dev_lease = netdev_get_by_index(net, ifindex_lease, &dev_tracker, 1388 GFP_KERNEL); 1389 if (!dev_lease) { 1390 err = -ENODEV; 1391 goto err_put_netns; 1392 } 1393 if (!netdev_can_lease_queue(dev_lease, info->extack)) { 1394 netdev_put(dev_lease, &dev_tracker); 1395 err = -EINVAL; 1396 goto err_put_netns; 1397 } 1398 1399 dev_lease = netdev_put_lock(dev_lease, net, &dev_tracker); 1400 if (!dev_lease) { 1401 err = -ENODEV; 1402 goto err_put_netns; 1403 } 1404 if (queue_id_lease >= dev_lease->real_num_rx_queues) { 1405 err = -ERANGE; 1406 NL_SET_BAD_ATTR(info->extack, qtb[NETDEV_A_QUEUE_ID]); 1407 goto err_unlock_dev_lease; 1408 } 1409 if (netdev_queue_busy(dev_lease, queue_id_lease, NETDEV_QUEUE_TYPE_RX, 1410 info->extack)) { 1411 err = -EBUSY; 1412 goto err_unlock_dev_lease; 1413 } 1414 1415 rxq_lease = __netif_get_rx_queue(dev_lease, queue_id_lease); 1416 rxq = __netif_get_rx_queue(dev, dev->real_num_rx_queues - 1); 1417 1418 /* Leasing queues from different physical devices is currently 1419 * not supported. Capabilities such as XDP features and DMA 1420 * device may differ between physical devices, and computing 1421 * a correct intersection for the virtual device is not yet 1422 * implemented. 1423 */ 1424 if (rxq->lease && rxq->lease->dev != dev_lease) { 1425 err = -EOPNOTSUPP; 1426 NL_SET_ERR_MSG(info->extack, 1427 "Leasing queues from different devices not supported"); 1428 goto err_unlock_dev_lease; 1429 } 1430 1431 queue_id = dev->queue_mgmt_ops->ndo_queue_create(dev, info->extack); 1432 if (queue_id < 0) { 1433 err = queue_id; 1434 goto err_unlock_dev_lease; 1435 } 1436 rxq = __netif_get_rx_queue(dev, queue_id); 1437 1438 netdev_rx_queue_lease(rxq, rxq_lease); 1439 1440 /* rsp was allocated large enough */ 1441 WARN_ON_ONCE(nla_put_u32(rsp, NETDEV_A_QUEUE_ID, queue_id)); 1442 1443 genlmsg_end(rsp, hdr); 1444 1445 netdev_unlock(dev_lease); 1446 netdev_unlock(dev); 1447 if (netns_lease >= 0) 1448 put_net(net); 1449 1450 return genlmsg_reply(rsp, info); 1451 1452 err_unlock_dev_lease: 1453 netdev_unlock(dev_lease); 1454 err_put_netns: 1455 if (netns_lease >= 0) 1456 put_net(net); 1457 err_unlock_dev: 1458 netdev_unlock(dev); 1459 err_genlmsg_free: 1460 nlmsg_free(rsp); 1461 return err; 1462 } 1463 1464 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv) 1465 { 1466 INIT_LIST_HEAD(&priv->bindings); 1467 mutex_init(&priv->lock); 1468 } 1469 1470 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv) 1471 { 1472 struct net_devmem_dmabuf_binding *binding; 1473 struct net_devmem_dmabuf_binding *temp; 1474 netdevice_tracker dev_tracker; 1475 struct net_device *dev; 1476 1477 mutex_lock(&priv->lock); 1478 list_for_each_entry_safe(binding, temp, &priv->bindings, list) { 1479 mutex_lock(&binding->lock); 1480 dev = binding->dev; 1481 if (!dev) { 1482 mutex_unlock(&binding->lock); 1483 net_devmem_unbind_dmabuf(binding); 1484 continue; 1485 } 1486 netdev_hold(dev, &dev_tracker, GFP_KERNEL); 1487 mutex_unlock(&binding->lock); 1488 1489 netdev_lock(dev); 1490 net_devmem_unbind_dmabuf(binding); 1491 netdev_unlock(dev); 1492 netdev_put(dev, &dev_tracker); 1493 } 1494 mutex_unlock(&priv->lock); 1495 } 1496 1497 static int netdev_genl_netdevice_event(struct notifier_block *nb, 1498 unsigned long event, void *ptr) 1499 { 1500 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 1501 1502 switch (event) { 1503 case NETDEV_REGISTER: 1504 netdev_lock_ops_to_full(netdev); 1505 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 1506 netdev_unlock_full_to_ops(netdev); 1507 break; 1508 case NETDEV_UNREGISTER: 1509 netdev_lock(netdev); 1510 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 1511 netdev_unlock(netdev); 1512 break; 1513 case NETDEV_XDP_FEAT_CHANGE: 1514 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 1515 break; 1516 } 1517 1518 return NOTIFY_OK; 1519 } 1520 1521 static struct notifier_block netdev_genl_nb = { 1522 .notifier_call = netdev_genl_netdevice_event, 1523 }; 1524 1525 static int __init netdev_genl_init(void) 1526 { 1527 int err; 1528 1529 err = register_netdevice_notifier(&netdev_genl_nb); 1530 if (err) 1531 return err; 1532 1533 err = genl_register_family(&netdev_nl_family); 1534 if (err) 1535 goto err_unreg_ntf; 1536 1537 return 0; 1538 1539 err_unreg_ntf: 1540 unregister_netdevice_notifier(&netdev_genl_nb); 1541 return err; 1542 } 1543 1544 subsys_initcall(netdev_genl_init); 1545