1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2023 Isovalent */ 3 4 #include <linux/netdevice.h> 5 #include <linux/ethtool.h> 6 #include <linux/etherdevice.h> 7 #include <linux/filter.h> 8 #include <linux/netfilter_netdev.h> 9 #include <linux/bpf_mprog.h> 10 #include <linux/indirect_call_wrapper.h> 11 12 #include <net/netdev_lock.h> 13 #include <net/netdev_queues.h> 14 #include <net/netdev_rx_queue.h> 15 #include <net/xdp_sock_drv.h> 16 #include <net/netkit.h> 17 #include <net/dst.h> 18 #include <net/tcx.h> 19 20 #define NETKIT_DRV_NAME "netkit" 21 22 #define NETKIT_NUM_RX_QUEUES_MAX 1024 23 #define NETKIT_NUM_TX_QUEUES_MAX 1 24 25 #define NETKIT_NUM_RX_QUEUES_REAL 1 26 #define NETKIT_NUM_TX_QUEUES_REAL 1 27 28 struct netkit { 29 __cacheline_group_begin(netkit_fastpath); 30 struct net_device __rcu *peer; 31 struct bpf_mprog_entry __rcu *active; 32 enum netkit_action policy; 33 enum netkit_scrub scrub; 34 struct bpf_mprog_bundle bundle; 35 __cacheline_group_end(netkit_fastpath); 36 37 __cacheline_group_begin(netkit_slowpath); 38 enum netkit_mode mode; 39 enum netkit_pairing pair; 40 bool primary; 41 u32 headroom; 42 __cacheline_group_end(netkit_slowpath); 43 }; 44 45 struct netkit_link { 46 struct bpf_link link; 47 struct net_device *dev; 48 }; 49 50 static struct rtnl_link_ops netkit_link_ops; 51 52 static __always_inline int 53 netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, 54 enum netkit_action ret) 55 { 56 const struct bpf_mprog_fp *fp; 57 const struct bpf_prog *prog; 58 59 bpf_mprog_foreach_prog(entry, fp, prog) { 60 bpf_compute_data_pointers(skb); 61 ret = bpf_prog_run(prog, skb); 62 if (ret != NETKIT_NEXT) 63 break; 64 } 65 return ret; 66 } 67 68 static void netkit_xnet(struct sk_buff *skb) 69 { 70 skb->priority = 0; 71 skb->mark = 0; 72 } 73 74 static void netkit_prep_forward(struct sk_buff *skb, 75 bool xnet, bool xnet_scrub) 76 { 77 skb_scrub_packet(skb, false); 78 nf_skip_egress(skb, true); 79 skb_reset_mac_header(skb); 80 if (!xnet) 81 return; 82 skb_clear_tstamp(skb); 83 if (xnet_scrub) 84 netkit_xnet(skb); 85 } 86 87 static struct netkit *netkit_priv(const struct net_device *dev) 88 { 89 return netdev_priv(dev); 90 } 91 92 static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev) 93 { 94 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 95 struct netkit *nk = netkit_priv(dev); 96 enum netkit_action ret = READ_ONCE(nk->policy); 97 netdev_tx_t ret_dev = NET_XMIT_SUCCESS; 98 const struct bpf_mprog_entry *entry; 99 struct net_device *peer; 100 int len = skb->len; 101 102 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 103 rcu_read_lock(); 104 peer = rcu_dereference(nk->peer); 105 if (unlikely(!peer || !(peer->flags & IFF_UP) || 106 !pskb_may_pull(skb, ETH_HLEN) || 107 skb_orphan_frags(skb, GFP_ATOMIC))) 108 goto drop; 109 netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), 110 nk->scrub); 111 eth_skb_pkt_type(skb, peer); 112 skb->dev = peer; 113 entry = rcu_dereference(nk->active); 114 if (entry) 115 ret = netkit_run(entry, skb, ret); 116 switch (ret) { 117 case NETKIT_NEXT: 118 case NETKIT_PASS: 119 eth_skb_pull_mac(skb); 120 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 121 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { 122 dev_sw_netstats_tx_add(dev, 1, len); 123 dev_sw_netstats_rx_add(peer, len); 124 } else { 125 goto drop_stats; 126 } 127 break; 128 case NETKIT_REDIRECT: 129 dev_sw_netstats_tx_add(dev, 1, len); 130 skb_do_redirect(skb); 131 break; 132 case NETKIT_DROP: 133 default: 134 drop: 135 kfree_skb(skb); 136 drop_stats: 137 dev_core_stats_tx_dropped_inc(dev); 138 ret_dev = NET_XMIT_DROP; 139 break; 140 } 141 rcu_read_unlock(); 142 bpf_net_ctx_clear(bpf_net_ctx); 143 return ret_dev; 144 } 145 146 static int netkit_open(struct net_device *dev) 147 { 148 struct netkit *nk = netkit_priv(dev); 149 struct net_device *peer = rtnl_dereference(nk->peer); 150 151 if (nk->pair == NETKIT_DEVICE_SINGLE) { 152 netif_carrier_on(dev); 153 return 0; 154 } 155 if (!peer) 156 return -ENOTCONN; 157 if (peer->flags & IFF_UP) { 158 netif_carrier_on(dev); 159 netif_carrier_on(peer); 160 } 161 return 0; 162 } 163 164 static int netkit_close(struct net_device *dev) 165 { 166 struct netkit *nk = netkit_priv(dev); 167 struct net_device *peer = rtnl_dereference(nk->peer); 168 169 netif_carrier_off(dev); 170 if (peer) 171 netif_carrier_off(peer); 172 return 0; 173 } 174 175 static int netkit_get_iflink(const struct net_device *dev) 176 { 177 struct netkit *nk = netkit_priv(dev); 178 struct net_device *peer; 179 int iflink = 0; 180 181 rcu_read_lock(); 182 peer = rcu_dereference(nk->peer); 183 if (peer) 184 iflink = READ_ONCE(peer->ifindex); 185 rcu_read_unlock(); 186 return iflink; 187 } 188 189 static void netkit_set_multicast(struct net_device *dev) 190 { 191 /* Nothing to do, we receive whatever gets pushed to us! */ 192 } 193 194 static int netkit_set_macaddr(struct net_device *dev, void *sa) 195 { 196 struct netkit *nk = netkit_priv(dev); 197 198 if (nk->mode != NETKIT_L2) 199 return -EOPNOTSUPP; 200 201 return eth_mac_addr(dev, sa); 202 } 203 204 static void netkit_set_headroom(struct net_device *dev, int headroom) 205 { 206 struct netkit *nk = netkit_priv(dev), *nk2; 207 struct net_device *peer; 208 209 if (headroom < 0) 210 headroom = NET_SKB_PAD; 211 212 rcu_read_lock(); 213 peer = rcu_dereference(nk->peer); 214 if (unlikely(!peer)) 215 goto out; 216 217 nk2 = netkit_priv(peer); 218 nk->headroom = headroom; 219 headroom = max(nk->headroom, nk2->headroom); 220 221 peer->needed_headroom = headroom; 222 dev->needed_headroom = headroom; 223 out: 224 rcu_read_unlock(); 225 } 226 227 INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev) 228 { 229 return rcu_dereference(netkit_priv(dev)->peer); 230 } 231 232 static void netkit_get_stats(struct net_device *dev, 233 struct rtnl_link_stats64 *stats) 234 { 235 dev_fetch_sw_netstats(stats, dev->tstats); 236 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 237 } 238 239 static bool netkit_xsk_supported_at_phys(const struct net_device *dev) 240 { 241 if (!dev->netdev_ops->ndo_bpf || 242 !dev->netdev_ops->ndo_xdp_xmit || 243 !dev->netdev_ops->ndo_xsk_wakeup) 244 return false; 245 if ((dev->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) 246 return false; 247 return true; 248 } 249 250 static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp) 251 { 252 struct netkit *nk = netkit_priv(dev); 253 struct netdev_bpf xdp_lower; 254 struct netdev_rx_queue *rxq; 255 struct net_device *phys; 256 int ret = -EBUSY; 257 258 switch (xdp->command) { 259 case XDP_SETUP_XSK_POOL: 260 if (nk->pair == NETKIT_DEVICE_PAIR) 261 return -EOPNOTSUPP; 262 if (xdp->xsk.queue_id >= dev->real_num_rx_queues) 263 return -EINVAL; 264 265 rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id); 266 if (!rxq->lease) 267 return -EOPNOTSUPP; 268 269 phys = rxq->lease->dev; 270 if (!netkit_xsk_supported_at_phys(phys)) 271 return -EOPNOTSUPP; 272 273 memcpy(&xdp_lower, xdp, sizeof(xdp_lower)); 274 xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease); 275 break; 276 case XDP_SETUP_PROG: 277 return -EPERM; 278 default: 279 return -EINVAL; 280 } 281 282 netdev_lock(phys); 283 if (!dev_get_min_mp_channel_count(phys)) 284 ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower); 285 netdev_unlock(phys); 286 return ret; 287 } 288 289 static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 290 { 291 struct netdev_rx_queue *rxq; 292 struct net_device *phys; 293 294 if (queue_id >= dev->real_num_rx_queues) 295 return -EINVAL; 296 297 rxq = __netif_get_rx_queue(dev, queue_id); 298 if (!rxq->lease) 299 return -EOPNOTSUPP; 300 301 phys = rxq->lease->dev; 302 if (!netkit_xsk_supported_at_phys(phys)) 303 return -EOPNOTSUPP; 304 305 return phys->netdev_ops->ndo_xsk_wakeup(phys, 306 get_netdev_rx_queue_index(rxq->lease), flags); 307 } 308 309 static int netkit_init(struct net_device *dev) 310 { 311 netdev_lockdep_set_classes(dev); 312 return 0; 313 } 314 315 static void netkit_uninit(struct net_device *dev); 316 317 static const struct net_device_ops netkit_netdev_ops = { 318 .ndo_init = netkit_init, 319 .ndo_open = netkit_open, 320 .ndo_stop = netkit_close, 321 .ndo_start_xmit = netkit_xmit, 322 .ndo_set_rx_mode = netkit_set_multicast, 323 .ndo_set_rx_headroom = netkit_set_headroom, 324 .ndo_set_mac_address = netkit_set_macaddr, 325 .ndo_get_iflink = netkit_get_iflink, 326 .ndo_get_peer_dev = netkit_peer_dev, 327 .ndo_get_stats64 = netkit_get_stats, 328 .ndo_uninit = netkit_uninit, 329 .ndo_bpf = netkit_xsk, 330 .ndo_xsk_wakeup = netkit_xsk_wakeup, 331 .ndo_features_check = passthru_features_check, 332 }; 333 334 static void netkit_get_drvinfo(struct net_device *dev, 335 struct ethtool_drvinfo *info) 336 { 337 strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver)); 338 } 339 340 static const struct ethtool_ops netkit_ethtool_ops = { 341 .get_drvinfo = netkit_get_drvinfo, 342 }; 343 344 static int netkit_queue_create(struct net_device *dev) 345 { 346 struct netkit *nk = netkit_priv(dev); 347 u32 rxq_count_old, rxq_count_new; 348 int err; 349 350 rxq_count_old = dev->real_num_rx_queues; 351 rxq_count_new = rxq_count_old + 1; 352 353 /* Only allow to lease a queue in single device mode or to 354 * lease against the peer device which then ends up in the 355 * target netns. 356 */ 357 if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) 358 return -EOPNOTSUPP; 359 360 if (netif_running(dev)) 361 netif_carrier_off(dev); 362 err = netif_set_real_num_rx_queues(dev, rxq_count_new); 363 if (netif_running(dev)) 364 netif_carrier_on(dev); 365 366 return err ? : rxq_count_old; 367 } 368 369 static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = { 370 .ndo_queue_create = netkit_queue_create, 371 }; 372 373 static struct net_device *netkit_alloc(struct nlattr *tb[], 374 const char *ifname, 375 unsigned char name_assign_type, 376 unsigned int num_tx_queues, 377 unsigned int num_rx_queues) 378 { 379 const struct rtnl_link_ops *ops = &netkit_link_ops; 380 struct net_device *dev; 381 382 if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX || 383 num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX) 384 return ERR_PTR(-EOPNOTSUPP); 385 386 dev = alloc_netdev_mqs(ops->priv_size, ifname, 387 name_assign_type, ops->setup, 388 num_tx_queues, num_rx_queues); 389 if (dev) { 390 dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL; 391 dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL; 392 } 393 return dev; 394 } 395 396 static void netkit_queue_unlease(struct net_device *dev) 397 { 398 struct netdev_rx_queue *rxq, *rxq_lease; 399 struct net_device *dev_lease; 400 int i; 401 402 if (dev->real_num_rx_queues == 1) 403 return; 404 405 netdev_lock(dev); 406 for (i = 1; i < dev->real_num_rx_queues; i++) { 407 rxq = __netif_get_rx_queue(dev, i); 408 rxq_lease = rxq->lease; 409 dev_lease = rxq_lease->dev; 410 411 netdev_lock(dev_lease); 412 netdev_rx_queue_unlease(rxq, rxq_lease); 413 netdev_unlock(dev_lease); 414 } 415 netdev_unlock(dev); 416 } 417 418 static void netkit_setup(struct net_device *dev) 419 { 420 static const netdev_features_t netkit_features_hw_vlan = 421 NETIF_F_HW_VLAN_CTAG_TX | 422 NETIF_F_HW_VLAN_CTAG_RX | 423 NETIF_F_HW_VLAN_STAG_TX | 424 NETIF_F_HW_VLAN_STAG_RX; 425 static const netdev_features_t netkit_features = 426 netkit_features_hw_vlan | 427 NETIF_F_SG | 428 NETIF_F_FRAGLIST | 429 NETIF_F_HW_CSUM | 430 NETIF_F_RXCSUM | 431 NETIF_F_SCTP_CRC | 432 NETIF_F_HIGHDMA | 433 NETIF_F_GSO_SOFTWARE | 434 NETIF_F_GSO_ENCAP_ALL; 435 436 ether_setup(dev); 437 dev->max_mtu = ETH_MAX_MTU; 438 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 439 440 dev->flags |= IFF_NOARP; 441 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 442 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 443 dev->priv_flags |= IFF_PHONY_HEADROOM; 444 dev->priv_flags |= IFF_NO_QUEUE; 445 dev->priv_flags |= IFF_DISABLE_NETPOLL; 446 dev->lltx = true; 447 448 dev->netdev_ops = &netkit_netdev_ops; 449 dev->ethtool_ops = &netkit_ethtool_ops; 450 dev->queue_mgmt_ops = &netkit_queue_mgmt_ops; 451 452 dev->features |= netkit_features; 453 dev->hw_features = netkit_features; 454 dev->hw_enc_features = netkit_features; 455 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 456 dev->vlan_features = dev->features & ~netkit_features_hw_vlan; 457 dev->needs_free_netdev = true; 458 459 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 460 461 xdp_set_features_flag(dev, NETDEV_XDP_ACT_XSK); 462 } 463 464 static struct net *netkit_get_link_net(const struct net_device *dev) 465 { 466 struct netkit *nk = netkit_priv(dev); 467 struct net_device *peer = rtnl_dereference(nk->peer); 468 469 return peer ? dev_net(peer) : dev_net(dev); 470 } 471 472 static int netkit_check_policy(int policy, struct nlattr *tb, 473 struct netlink_ext_ack *extack) 474 { 475 switch (policy) { 476 case NETKIT_PASS: 477 case NETKIT_DROP: 478 return 0; 479 default: 480 NL_SET_ERR_MSG_ATTR(extack, tb, 481 "Provided default xmit policy not supported"); 482 return -EINVAL; 483 } 484 } 485 486 static int netkit_validate(struct nlattr *tb[], struct nlattr *data[], 487 struct netlink_ext_ack *extack) 488 { 489 struct nlattr *attr = tb[IFLA_ADDRESS]; 490 491 if (!attr) 492 return 0; 493 if (nla_len(attr) != ETH_ALEN) 494 return -EINVAL; 495 if (!is_valid_ether_addr(nla_data(attr))) 496 return -EADDRNOTAVAIL; 497 return 0; 498 } 499 500 static int netkit_new_link(struct net_device *dev, 501 struct rtnl_newlink_params *params, 502 struct netlink_ext_ack *extack) 503 { 504 struct net *peer_net = rtnl_newlink_peer_net(params); 505 enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT; 506 enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT; 507 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr; 508 enum netkit_pairing pair = NETKIT_DEVICE_PAIR; 509 enum netkit_action policy_prim = NETKIT_PASS; 510 enum netkit_action policy_peer = NETKIT_PASS; 511 struct nlattr **data = params->data; 512 enum netkit_mode mode = NETKIT_L3; 513 unsigned char ifname_assign_type; 514 struct nlattr **tb = params->tb; 515 u16 headroom = 0, tailroom = 0; 516 struct ifinfomsg *ifmp = NULL; 517 struct net_device *peer = NULL; 518 bool seen_peer = false; 519 char ifname[IFNAMSIZ]; 520 struct netkit *nk; 521 int err; 522 523 tbp = tb; 524 if (data) { 525 if (data[IFLA_NETKIT_MODE]) 526 mode = nla_get_u32(data[IFLA_NETKIT_MODE]); 527 if (data[IFLA_NETKIT_PEER_INFO]) { 528 attr = data[IFLA_NETKIT_PEER_INFO]; 529 ifmp = nla_data(attr); 530 rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack); 531 tbp = peer_tb; 532 } 533 if (data[IFLA_NETKIT_SCRUB]) 534 scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]); 535 if (data[IFLA_NETKIT_PEER_SCRUB]) 536 scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]); 537 if (data[IFLA_NETKIT_POLICY]) { 538 attr = data[IFLA_NETKIT_POLICY]; 539 policy_prim = nla_get_u32(attr); 540 err = netkit_check_policy(policy_prim, attr, extack); 541 if (err < 0) 542 return err; 543 } 544 if (data[IFLA_NETKIT_PEER_POLICY]) { 545 attr = data[IFLA_NETKIT_PEER_POLICY]; 546 policy_peer = nla_get_u32(attr); 547 err = netkit_check_policy(policy_peer, attr, extack); 548 if (err < 0) 549 return err; 550 } 551 if (data[IFLA_NETKIT_HEADROOM]) 552 headroom = nla_get_u16(data[IFLA_NETKIT_HEADROOM]); 553 if (data[IFLA_NETKIT_TAILROOM]) 554 tailroom = nla_get_u16(data[IFLA_NETKIT_TAILROOM]); 555 if (data[IFLA_NETKIT_PAIRING]) 556 pair = nla_get_u32(data[IFLA_NETKIT_PAIRING]); 557 558 seen_peer = data[IFLA_NETKIT_PEER_INFO] || 559 data[IFLA_NETKIT_PEER_SCRUB] || 560 data[IFLA_NETKIT_PEER_POLICY]; 561 } 562 563 if (ifmp && tbp[IFLA_IFNAME]) { 564 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 565 ifname_assign_type = NET_NAME_USER; 566 } else { 567 strscpy(ifname, "nk%d", IFNAMSIZ); 568 ifname_assign_type = NET_NAME_ENUM; 569 } 570 if (mode != NETKIT_L2 && 571 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS])) 572 return -EOPNOTSUPP; 573 if (pair == NETKIT_DEVICE_SINGLE && 574 (tb != tbp || seen_peer || policy_prim != NETKIT_PASS)) 575 return -EOPNOTSUPP; 576 577 if (pair == NETKIT_DEVICE_PAIR) { 578 peer = rtnl_create_link(peer_net, ifname, ifname_assign_type, 579 &netkit_link_ops, tbp, extack); 580 if (IS_ERR(peer)) 581 return PTR_ERR(peer); 582 583 netif_inherit_tso_max(peer, dev); 584 if (headroom) 585 peer->needed_headroom = headroom; 586 if (tailroom) 587 peer->needed_tailroom = tailroom; 588 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS])) 589 eth_hw_addr_random(peer); 590 if (ifmp && dev->ifindex) 591 peer->ifindex = ifmp->ifi_index; 592 593 nk = netkit_priv(peer); 594 nk->primary = false; 595 nk->policy = policy_peer; 596 nk->scrub = scrub_peer; 597 nk->mode = mode; 598 nk->pair = pair; 599 nk->headroom = headroom; 600 bpf_mprog_bundle_init(&nk->bundle); 601 602 err = register_netdevice(peer); 603 if (err < 0) 604 goto err_register_peer; 605 netif_carrier_off(peer); 606 if (mode == NETKIT_L2) 607 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL); 608 609 err = rtnl_configure_link(peer, NULL, 0, NULL); 610 if (err < 0) 611 goto err_configure_peer; 612 } 613 614 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS]) 615 eth_hw_addr_random(dev); 616 if (tb[IFLA_IFNAME]) 617 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 618 else 619 strscpy(dev->name, "nk%d", IFNAMSIZ); 620 if (headroom) 621 dev->needed_headroom = headroom; 622 if (tailroom) 623 dev->needed_tailroom = tailroom; 624 625 nk = netkit_priv(dev); 626 nk->primary = true; 627 nk->policy = policy_prim; 628 nk->scrub = scrub_prim; 629 nk->mode = mode; 630 nk->pair = pair; 631 nk->headroom = headroom; 632 bpf_mprog_bundle_init(&nk->bundle); 633 634 err = register_netdevice(dev); 635 if (err < 0) 636 goto err_configure_peer; 637 netif_carrier_off(dev); 638 if (mode == NETKIT_L2) 639 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL); 640 641 rcu_assign_pointer(netkit_priv(dev)->peer, peer); 642 if (peer) 643 rcu_assign_pointer(netkit_priv(peer)->peer, dev); 644 return 0; 645 err_configure_peer: 646 if (peer) 647 unregister_netdevice(peer); 648 return err; 649 err_register_peer: 650 free_netdev(peer); 651 return err; 652 } 653 654 static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev, 655 bool bundle_fallback) 656 { 657 struct netkit *nk = netkit_priv(dev); 658 struct bpf_mprog_entry *entry; 659 660 ASSERT_RTNL(); 661 entry = rcu_dereference_rtnl(nk->active); 662 if (entry) 663 return entry; 664 if (bundle_fallback) 665 return &nk->bundle.a; 666 return NULL; 667 } 668 669 static void netkit_entry_update(struct net_device *dev, 670 struct bpf_mprog_entry *entry) 671 { 672 struct netkit *nk = netkit_priv(dev); 673 674 ASSERT_RTNL(); 675 rcu_assign_pointer(nk->active, entry); 676 } 677 678 static void netkit_entry_sync(void) 679 { 680 synchronize_rcu(); 681 } 682 683 static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which) 684 { 685 struct net_device *dev; 686 struct netkit *nk; 687 688 ASSERT_RTNL(); 689 690 switch (which) { 691 case BPF_NETKIT_PRIMARY: 692 case BPF_NETKIT_PEER: 693 break; 694 default: 695 return ERR_PTR(-EINVAL); 696 } 697 698 dev = __dev_get_by_index(net, ifindex); 699 if (!dev) 700 return ERR_PTR(-ENODEV); 701 if (dev->netdev_ops != &netkit_netdev_ops) 702 return ERR_PTR(-ENXIO); 703 704 nk = netkit_priv(dev); 705 if (!nk->primary) 706 return ERR_PTR(-EACCES); 707 if (nk->pair == NETKIT_DEVICE_SINGLE) 708 return ERR_PTR(-EOPNOTSUPP); 709 if (which == BPF_NETKIT_PEER) { 710 dev = rcu_dereference_rtnl(nk->peer); 711 if (!dev) 712 return ERR_PTR(-ENODEV); 713 } 714 return dev; 715 } 716 717 int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) 718 { 719 struct bpf_mprog_entry *entry, *entry_new; 720 struct bpf_prog *replace_prog = NULL; 721 struct net_device *dev; 722 int ret; 723 724 rtnl_lock(); 725 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 726 attr->attach_type); 727 if (IS_ERR(dev)) { 728 ret = PTR_ERR(dev); 729 goto out; 730 } 731 entry = netkit_entry_fetch(dev, true); 732 if (attr->attach_flags & BPF_F_REPLACE) { 733 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, 734 prog->type); 735 if (IS_ERR(replace_prog)) { 736 ret = PTR_ERR(replace_prog); 737 replace_prog = NULL; 738 goto out; 739 } 740 } 741 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, 742 attr->attach_flags, attr->relative_fd, 743 attr->expected_revision); 744 if (!ret) { 745 if (entry != entry_new) { 746 netkit_entry_update(dev, entry_new); 747 netkit_entry_sync(); 748 } 749 bpf_mprog_commit(entry); 750 } 751 out: 752 if (replace_prog) 753 bpf_prog_put(replace_prog); 754 rtnl_unlock(); 755 return ret; 756 } 757 758 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) 759 { 760 struct bpf_mprog_entry *entry, *entry_new; 761 struct net_device *dev; 762 int ret; 763 764 rtnl_lock(); 765 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 766 attr->attach_type); 767 if (IS_ERR(dev)) { 768 ret = PTR_ERR(dev); 769 goto out; 770 } 771 entry = netkit_entry_fetch(dev, false); 772 if (!entry) { 773 ret = -ENOENT; 774 goto out; 775 } 776 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, 777 attr->relative_fd, attr->expected_revision); 778 if (!ret) { 779 if (!bpf_mprog_total(entry_new)) 780 entry_new = NULL; 781 netkit_entry_update(dev, entry_new); 782 netkit_entry_sync(); 783 bpf_mprog_commit(entry); 784 } 785 out: 786 rtnl_unlock(); 787 return ret; 788 } 789 790 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) 791 { 792 struct net_device *dev; 793 int ret; 794 795 rtnl_lock(); 796 dev = netkit_dev_fetch(current->nsproxy->net_ns, 797 attr->query.target_ifindex, 798 attr->query.attach_type); 799 if (IS_ERR(dev)) { 800 ret = PTR_ERR(dev); 801 goto out; 802 } 803 ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false)); 804 out: 805 rtnl_unlock(); 806 return ret; 807 } 808 809 static struct netkit_link *netkit_link(const struct bpf_link *link) 810 { 811 return container_of(link, struct netkit_link, link); 812 } 813 814 static int netkit_link_prog_attach(struct bpf_link *link, u32 flags, 815 u32 id_or_fd, u64 revision) 816 { 817 struct netkit_link *nkl = netkit_link(link); 818 struct bpf_mprog_entry *entry, *entry_new; 819 struct net_device *dev = nkl->dev; 820 int ret; 821 822 ASSERT_RTNL(); 823 entry = netkit_entry_fetch(dev, true); 824 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, 825 id_or_fd, revision); 826 if (!ret) { 827 if (entry != entry_new) { 828 netkit_entry_update(dev, entry_new); 829 netkit_entry_sync(); 830 } 831 bpf_mprog_commit(entry); 832 } 833 return ret; 834 } 835 836 static void netkit_link_release(struct bpf_link *link) 837 { 838 struct netkit_link *nkl = netkit_link(link); 839 struct bpf_mprog_entry *entry, *entry_new; 840 struct net_device *dev; 841 int ret = 0; 842 843 rtnl_lock(); 844 dev = nkl->dev; 845 if (!dev) 846 goto out; 847 entry = netkit_entry_fetch(dev, false); 848 if (!entry) { 849 ret = -ENOENT; 850 goto out; 851 } 852 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); 853 if (!ret) { 854 if (!bpf_mprog_total(entry_new)) 855 entry_new = NULL; 856 netkit_entry_update(dev, entry_new); 857 netkit_entry_sync(); 858 bpf_mprog_commit(entry); 859 nkl->dev = NULL; 860 } 861 out: 862 WARN_ON_ONCE(ret); 863 rtnl_unlock(); 864 } 865 866 static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog, 867 struct bpf_prog *oprog) 868 { 869 struct netkit_link *nkl = netkit_link(link); 870 struct bpf_mprog_entry *entry, *entry_new; 871 struct net_device *dev; 872 int ret = 0; 873 874 rtnl_lock(); 875 dev = nkl->dev; 876 if (!dev) { 877 ret = -ENOLINK; 878 goto out; 879 } 880 if (oprog && link->prog != oprog) { 881 ret = -EPERM; 882 goto out; 883 } 884 oprog = link->prog; 885 if (oprog == nprog) { 886 bpf_prog_put(nprog); 887 goto out; 888 } 889 entry = netkit_entry_fetch(dev, false); 890 if (!entry) { 891 ret = -ENOENT; 892 goto out; 893 } 894 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, 895 BPF_F_REPLACE | BPF_F_ID, 896 link->prog->aux->id, 0); 897 if (!ret) { 898 WARN_ON_ONCE(entry != entry_new); 899 oprog = xchg(&link->prog, nprog); 900 bpf_prog_put(oprog); 901 bpf_mprog_commit(entry); 902 } 903 out: 904 rtnl_unlock(); 905 return ret; 906 } 907 908 static void netkit_link_dealloc(struct bpf_link *link) 909 { 910 kfree(netkit_link(link)); 911 } 912 913 static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) 914 { 915 const struct netkit_link *nkl = netkit_link(link); 916 u32 ifindex = 0; 917 918 rtnl_lock(); 919 if (nkl->dev) 920 ifindex = nkl->dev->ifindex; 921 rtnl_unlock(); 922 923 seq_printf(seq, "ifindex:\t%u\n", ifindex); 924 seq_printf(seq, "attach_type:\t%u (%s)\n", 925 link->attach_type, 926 link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer"); 927 } 928 929 static int netkit_link_fill_info(const struct bpf_link *link, 930 struct bpf_link_info *info) 931 { 932 const struct netkit_link *nkl = netkit_link(link); 933 u32 ifindex = 0; 934 935 rtnl_lock(); 936 if (nkl->dev) 937 ifindex = nkl->dev->ifindex; 938 rtnl_unlock(); 939 940 info->netkit.ifindex = ifindex; 941 info->netkit.attach_type = link->attach_type; 942 return 0; 943 } 944 945 static int netkit_link_detach(struct bpf_link *link) 946 { 947 netkit_link_release(link); 948 return 0; 949 } 950 951 static const struct bpf_link_ops netkit_link_lops = { 952 .release = netkit_link_release, 953 .detach = netkit_link_detach, 954 .dealloc = netkit_link_dealloc, 955 .update_prog = netkit_link_update, 956 .show_fdinfo = netkit_link_fdinfo, 957 .fill_link_info = netkit_link_fill_info, 958 }; 959 960 static int netkit_link_init(struct netkit_link *nkl, 961 struct bpf_link_primer *link_primer, 962 const union bpf_attr *attr, 963 struct net_device *dev, 964 struct bpf_prog *prog) 965 { 966 bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT, 967 &netkit_link_lops, prog, attr->link_create.attach_type); 968 nkl->dev = dev; 969 return bpf_link_prime(&nkl->link, link_primer); 970 } 971 972 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 973 { 974 struct bpf_link_primer link_primer; 975 struct netkit_link *nkl; 976 struct net_device *dev; 977 int ret; 978 979 rtnl_lock(); 980 dev = netkit_dev_fetch(current->nsproxy->net_ns, 981 attr->link_create.target_ifindex, 982 attr->link_create.attach_type); 983 if (IS_ERR(dev)) { 984 ret = PTR_ERR(dev); 985 goto out; 986 } 987 nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT); 988 if (!nkl) { 989 ret = -ENOMEM; 990 goto out; 991 } 992 ret = netkit_link_init(nkl, &link_primer, attr, dev, prog); 993 if (ret) { 994 kfree(nkl); 995 goto out; 996 } 997 ret = netkit_link_prog_attach(&nkl->link, 998 attr->link_create.flags, 999 attr->link_create.netkit.relative_fd, 1000 attr->link_create.netkit.expected_revision); 1001 if (ret) { 1002 nkl->dev = NULL; 1003 bpf_link_cleanup(&link_primer); 1004 goto out; 1005 } 1006 ret = bpf_link_settle(&link_primer); 1007 out: 1008 rtnl_unlock(); 1009 return ret; 1010 } 1011 1012 static void netkit_release_all(struct net_device *dev) 1013 { 1014 struct bpf_mprog_entry *entry; 1015 struct bpf_tuple tuple = {}; 1016 struct bpf_mprog_fp *fp; 1017 struct bpf_mprog_cp *cp; 1018 1019 entry = netkit_entry_fetch(dev, false); 1020 if (!entry) 1021 return; 1022 netkit_entry_update(dev, NULL); 1023 netkit_entry_sync(); 1024 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { 1025 if (tuple.link) 1026 netkit_link(tuple.link)->dev = NULL; 1027 else 1028 bpf_prog_put(tuple.prog); 1029 } 1030 } 1031 1032 static void netkit_uninit(struct net_device *dev) 1033 { 1034 netkit_release_all(dev); 1035 netkit_queue_unlease(dev); 1036 } 1037 1038 static void netkit_del_link(struct net_device *dev, struct list_head *head) 1039 { 1040 struct netkit *nk = netkit_priv(dev); 1041 struct net_device *peer = rtnl_dereference(nk->peer); 1042 1043 RCU_INIT_POINTER(nk->peer, NULL); 1044 unregister_netdevice_queue(dev, head); 1045 if (peer) { 1046 nk = netkit_priv(peer); 1047 RCU_INIT_POINTER(nk->peer, NULL); 1048 unregister_netdevice_queue(peer, head); 1049 } 1050 } 1051 1052 static int netkit_change_link(struct net_device *dev, struct nlattr *tb[], 1053 struct nlattr *data[], 1054 struct netlink_ext_ack *extack) 1055 { 1056 struct netkit *nk = netkit_priv(dev); 1057 struct net_device *peer = rtnl_dereference(nk->peer); 1058 enum netkit_action policy; 1059 struct nlattr *attr; 1060 int err, i; 1061 static const struct { 1062 u32 attr; 1063 char *name; 1064 } fixed_params[] = { 1065 { IFLA_NETKIT_MODE, "operating mode" }, 1066 { IFLA_NETKIT_SCRUB, "scrubbing" }, 1067 { IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" }, 1068 { IFLA_NETKIT_PEER_INFO, "peer info" }, 1069 { IFLA_NETKIT_HEADROOM, "headroom" }, 1070 { IFLA_NETKIT_TAILROOM, "tailroom" }, 1071 { IFLA_NETKIT_PAIRING, "pairing" }, 1072 }; 1073 1074 if (!nk->primary) { 1075 NL_SET_ERR_MSG(extack, 1076 "netkit link settings can be changed only through the primary device"); 1077 return -EACCES; 1078 } 1079 1080 for (i = 0; i < ARRAY_SIZE(fixed_params); i++) { 1081 attr = data[fixed_params[i].attr]; 1082 if (attr) { 1083 NL_SET_ERR_MSG_ATTR_FMT(extack, attr, 1084 "netkit link %s cannot be changed after device creation", 1085 fixed_params[i].name); 1086 return -EACCES; 1087 } 1088 } 1089 1090 if (data[IFLA_NETKIT_POLICY]) { 1091 err = -EOPNOTSUPP; 1092 attr = data[IFLA_NETKIT_POLICY]; 1093 policy = nla_get_u32(attr); 1094 if (nk->pair == NETKIT_DEVICE_PAIR) 1095 err = netkit_check_policy(policy, attr, extack); 1096 if (err) 1097 return err; 1098 WRITE_ONCE(nk->policy, policy); 1099 } 1100 1101 if (data[IFLA_NETKIT_PEER_POLICY]) { 1102 err = -EOPNOTSUPP; 1103 attr = data[IFLA_NETKIT_PEER_POLICY]; 1104 policy = nla_get_u32(attr); 1105 if (peer) 1106 err = netkit_check_policy(policy, attr, extack); 1107 if (err) 1108 return err; 1109 nk = netkit_priv(peer); 1110 WRITE_ONCE(nk->policy, policy); 1111 } 1112 1113 return 0; 1114 } 1115 1116 static void netkit_check_lease_unregister(struct net_device *dev) 1117 { 1118 LIST_HEAD(list_kill); 1119 u32 q_idx; 1120 1121 if (READ_ONCE(dev->reg_state) != NETREG_UNREGISTERING || 1122 !dev->dev.parent) 1123 return; 1124 1125 netdev_lock_ops(dev); 1126 for (q_idx = 0; q_idx < dev->real_num_rx_queues; q_idx++) { 1127 struct net_device *tmp = dev; 1128 u32 tmp_q_idx = q_idx; 1129 1130 if (netif_rx_queue_lease_get_owner(&tmp, &tmp_q_idx)) { 1131 if (tmp->netdev_ops != &netkit_netdev_ops) 1132 continue; 1133 /* A single phys device can have multiple queues leased 1134 * to one netkit device. We can only queue that netkit 1135 * device once to the list_kill. Queues of that phys 1136 * device can be leased with different individual netkit 1137 * devices, hence we batch via list_kill. 1138 */ 1139 if (unregister_netdevice_queued(tmp)) 1140 continue; 1141 netkit_del_link(tmp, &list_kill); 1142 } 1143 } 1144 netdev_unlock_ops(dev); 1145 unregister_netdevice_many(&list_kill); 1146 } 1147 1148 static int netkit_notifier(struct notifier_block *this, 1149 unsigned long event, void *ptr) 1150 { 1151 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 1152 1153 if (event == NETDEV_UNREGISTER) 1154 netkit_check_lease_unregister(dev); 1155 return NOTIFY_DONE; 1156 } 1157 1158 static size_t netkit_get_size(const struct net_device *dev) 1159 { 1160 return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */ 1161 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */ 1162 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */ 1163 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */ 1164 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */ 1165 nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */ 1166 nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */ 1167 nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */ 1168 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PAIRING */ 1169 0; 1170 } 1171 1172 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev) 1173 { 1174 struct netkit *nk = netkit_priv(dev); 1175 struct net_device *peer = rtnl_dereference(nk->peer); 1176 1177 if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary)) 1178 return -EMSGSIZE; 1179 if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy)) 1180 return -EMSGSIZE; 1181 if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode)) 1182 return -EMSGSIZE; 1183 if (nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub)) 1184 return -EMSGSIZE; 1185 if (nla_put_u16(skb, IFLA_NETKIT_HEADROOM, dev->needed_headroom)) 1186 return -EMSGSIZE; 1187 if (nla_put_u16(skb, IFLA_NETKIT_TAILROOM, dev->needed_tailroom)) 1188 return -EMSGSIZE; 1189 if (nla_put_u32(skb, IFLA_NETKIT_PAIRING, nk->pair)) 1190 return -EMSGSIZE; 1191 1192 if (peer) { 1193 nk = netkit_priv(peer); 1194 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy)) 1195 return -EMSGSIZE; 1196 if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub)) 1197 return -EMSGSIZE; 1198 } 1199 1200 return 0; 1201 } 1202 1203 static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = { 1204 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) }, 1205 [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3), 1206 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 }, 1207 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 }, 1208 [IFLA_NETKIT_HEADROOM] = { .type = NLA_U16 }, 1209 [IFLA_NETKIT_TAILROOM] = { .type = NLA_U16 }, 1210 [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), 1211 [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), 1212 [IFLA_NETKIT_PAIRING] = NLA_POLICY_MAX(NLA_U32, NETKIT_DEVICE_SINGLE), 1213 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT, 1214 .reject_message = "Primary attribute is read-only" }, 1215 }; 1216 1217 static struct rtnl_link_ops netkit_link_ops = { 1218 .kind = NETKIT_DRV_NAME, 1219 .priv_size = sizeof(struct netkit), 1220 .alloc = netkit_alloc, 1221 .setup = netkit_setup, 1222 .newlink = netkit_new_link, 1223 .dellink = netkit_del_link, 1224 .changelink = netkit_change_link, 1225 .get_link_net = netkit_get_link_net, 1226 .get_size = netkit_get_size, 1227 .fill_info = netkit_fill_info, 1228 .policy = netkit_policy, 1229 .validate = netkit_validate, 1230 .peer_type = IFLA_NETKIT_PEER_INFO, 1231 .maxtype = IFLA_NETKIT_MAX, 1232 }; 1233 1234 static struct notifier_block netkit_netdev_notifier = { 1235 .notifier_call = netkit_notifier, 1236 }; 1237 1238 static __init int netkit_mod_init(void) 1239 { 1240 int ret; 1241 1242 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT || 1243 (int)NETKIT_PASS != (int)TCX_PASS || 1244 (int)NETKIT_DROP != (int)TCX_DROP || 1245 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT); 1246 1247 ret = rtnl_link_register(&netkit_link_ops); 1248 if (ret) 1249 return ret; 1250 ret = register_netdevice_notifier(&netkit_netdev_notifier); 1251 if (ret) 1252 rtnl_link_unregister(&netkit_link_ops); 1253 return ret; 1254 } 1255 1256 static __exit void netkit_mod_exit(void) 1257 { 1258 unregister_netdevice_notifier(&netkit_netdev_notifier); 1259 rtnl_link_unregister(&netkit_link_ops); 1260 } 1261 1262 module_init(netkit_mod_init); 1263 module_exit(netkit_mod_exit); 1264 1265 MODULE_DESCRIPTION("BPF-programmable network device"); 1266 MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>"); 1267 MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>"); 1268 MODULE_LICENSE("GPL"); 1269 MODULE_ALIAS_RTNL_LINK(NETKIT_DRV_NAME); 1270