1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2023 Isovalent */ 3 4 #include <linux/netdevice.h> 5 #include <linux/ethtool.h> 6 #include <linux/etherdevice.h> 7 #include <linux/filter.h> 8 #include <linux/netfilter_netdev.h> 9 #include <linux/bpf_mprog.h> 10 #include <linux/indirect_call_wrapper.h> 11 12 #include <net/netkit.h> 13 #include <net/dst.h> 14 #include <net/tcx.h> 15 16 #define DRV_NAME "netkit" 17 18 struct netkit { 19 /* Needed in fast-path */ 20 struct net_device __rcu *peer; 21 struct bpf_mprog_entry __rcu *active; 22 enum netkit_action policy; 23 struct bpf_mprog_bundle bundle; 24 25 /* Needed in slow-path */ 26 enum netkit_mode mode; 27 bool primary; 28 u32 headroom; 29 }; 30 31 struct netkit_link { 32 struct bpf_link link; 33 struct net_device *dev; 34 u32 location; 35 }; 36 37 static __always_inline int 38 netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, 39 enum netkit_action ret) 40 { 41 const struct bpf_mprog_fp *fp; 42 const struct bpf_prog *prog; 43 44 bpf_mprog_foreach_prog(entry, fp, prog) { 45 bpf_compute_data_pointers(skb); 46 ret = bpf_prog_run(prog, skb); 47 if (ret != NETKIT_NEXT) 48 break; 49 } 50 return ret; 51 } 52 53 static void netkit_prep_forward(struct sk_buff *skb, bool xnet) 54 { 55 skb_scrub_packet(skb, xnet); 56 skb->priority = 0; 57 nf_skip_egress(skb, true); 58 skb_reset_mac_header(skb); 59 } 60 61 static struct netkit *netkit_priv(const struct net_device *dev) 62 { 63 return netdev_priv(dev); 64 } 65 66 static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev) 67 { 68 struct netkit *nk = netkit_priv(dev); 69 enum netkit_action ret = READ_ONCE(nk->policy); 70 netdev_tx_t ret_dev = NET_XMIT_SUCCESS; 71 const struct bpf_mprog_entry *entry; 72 struct net_device *peer; 73 int len = skb->len; 74 75 rcu_read_lock(); 76 peer = rcu_dereference(nk->peer); 77 if (unlikely(!peer || !(peer->flags & IFF_UP) || 78 !pskb_may_pull(skb, ETH_HLEN) || 79 skb_orphan_frags(skb, GFP_ATOMIC))) 80 goto drop; 81 netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer))); 82 eth_skb_pkt_type(skb, peer); 83 skb->dev = peer; 84 entry = rcu_dereference(nk->active); 85 if (entry) 86 ret = netkit_run(entry, skb, ret); 87 switch (ret) { 88 case NETKIT_NEXT: 89 case NETKIT_PASS: 90 eth_skb_pull_mac(skb); 91 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 92 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { 93 dev_sw_netstats_tx_add(dev, 1, len); 94 dev_sw_netstats_rx_add(peer, len); 95 } else { 96 goto drop_stats; 97 } 98 break; 99 case NETKIT_REDIRECT: 100 dev_sw_netstats_tx_add(dev, 1, len); 101 skb_do_redirect(skb); 102 break; 103 case NETKIT_DROP: 104 default: 105 drop: 106 kfree_skb(skb); 107 drop_stats: 108 dev_core_stats_tx_dropped_inc(dev); 109 ret_dev = NET_XMIT_DROP; 110 break; 111 } 112 rcu_read_unlock(); 113 return ret_dev; 114 } 115 116 static int netkit_open(struct net_device *dev) 117 { 118 struct netkit *nk = netkit_priv(dev); 119 struct net_device *peer = rtnl_dereference(nk->peer); 120 121 if (!peer) 122 return -ENOTCONN; 123 if (peer->flags & IFF_UP) { 124 netif_carrier_on(dev); 125 netif_carrier_on(peer); 126 } 127 return 0; 128 } 129 130 static int netkit_close(struct net_device *dev) 131 { 132 struct netkit *nk = netkit_priv(dev); 133 struct net_device *peer = rtnl_dereference(nk->peer); 134 135 netif_carrier_off(dev); 136 if (peer) 137 netif_carrier_off(peer); 138 return 0; 139 } 140 141 static int netkit_get_iflink(const struct net_device *dev) 142 { 143 struct netkit *nk = netkit_priv(dev); 144 struct net_device *peer; 145 int iflink = 0; 146 147 rcu_read_lock(); 148 peer = rcu_dereference(nk->peer); 149 if (peer) 150 iflink = READ_ONCE(peer->ifindex); 151 rcu_read_unlock(); 152 return iflink; 153 } 154 155 static void netkit_set_multicast(struct net_device *dev) 156 { 157 /* Nothing to do, we receive whatever gets pushed to us! */ 158 } 159 160 static int netkit_set_macaddr(struct net_device *dev, void *sa) 161 { 162 struct netkit *nk = netkit_priv(dev); 163 164 if (nk->mode != NETKIT_L2) 165 return -EOPNOTSUPP; 166 167 return eth_mac_addr(dev, sa); 168 } 169 170 static void netkit_set_headroom(struct net_device *dev, int headroom) 171 { 172 struct netkit *nk = netkit_priv(dev), *nk2; 173 struct net_device *peer; 174 175 if (headroom < 0) 176 headroom = NET_SKB_PAD; 177 178 rcu_read_lock(); 179 peer = rcu_dereference(nk->peer); 180 if (unlikely(!peer)) 181 goto out; 182 183 nk2 = netkit_priv(peer); 184 nk->headroom = headroom; 185 headroom = max(nk->headroom, nk2->headroom); 186 187 peer->needed_headroom = headroom; 188 dev->needed_headroom = headroom; 189 out: 190 rcu_read_unlock(); 191 } 192 193 INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev) 194 { 195 return rcu_dereference(netkit_priv(dev)->peer); 196 } 197 198 static void netkit_get_stats(struct net_device *dev, 199 struct rtnl_link_stats64 *stats) 200 { 201 dev_fetch_sw_netstats(stats, dev->tstats); 202 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 203 } 204 205 static void netkit_uninit(struct net_device *dev); 206 207 static const struct net_device_ops netkit_netdev_ops = { 208 .ndo_open = netkit_open, 209 .ndo_stop = netkit_close, 210 .ndo_start_xmit = netkit_xmit, 211 .ndo_set_rx_mode = netkit_set_multicast, 212 .ndo_set_rx_headroom = netkit_set_headroom, 213 .ndo_set_mac_address = netkit_set_macaddr, 214 .ndo_get_iflink = netkit_get_iflink, 215 .ndo_get_peer_dev = netkit_peer_dev, 216 .ndo_get_stats64 = netkit_get_stats, 217 .ndo_uninit = netkit_uninit, 218 .ndo_features_check = passthru_features_check, 219 }; 220 221 static void netkit_get_drvinfo(struct net_device *dev, 222 struct ethtool_drvinfo *info) 223 { 224 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 225 } 226 227 static const struct ethtool_ops netkit_ethtool_ops = { 228 .get_drvinfo = netkit_get_drvinfo, 229 }; 230 231 static void netkit_setup(struct net_device *dev) 232 { 233 static const netdev_features_t netkit_features_hw_vlan = 234 NETIF_F_HW_VLAN_CTAG_TX | 235 NETIF_F_HW_VLAN_CTAG_RX | 236 NETIF_F_HW_VLAN_STAG_TX | 237 NETIF_F_HW_VLAN_STAG_RX; 238 static const netdev_features_t netkit_features = 239 netkit_features_hw_vlan | 240 NETIF_F_SG | 241 NETIF_F_FRAGLIST | 242 NETIF_F_HW_CSUM | 243 NETIF_F_RXCSUM | 244 NETIF_F_SCTP_CRC | 245 NETIF_F_HIGHDMA | 246 NETIF_F_GSO_SOFTWARE | 247 NETIF_F_GSO_ENCAP_ALL; 248 249 ether_setup(dev); 250 dev->max_mtu = ETH_MAX_MTU; 251 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 252 253 dev->flags |= IFF_NOARP; 254 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 255 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 256 dev->priv_flags |= IFF_PHONY_HEADROOM; 257 dev->priv_flags |= IFF_NO_QUEUE; 258 dev->priv_flags |= IFF_DISABLE_NETPOLL; 259 dev->lltx = true; 260 261 dev->ethtool_ops = &netkit_ethtool_ops; 262 dev->netdev_ops = &netkit_netdev_ops; 263 264 dev->features |= netkit_features; 265 dev->hw_features = netkit_features; 266 dev->hw_enc_features = netkit_features; 267 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 268 dev->vlan_features = dev->features & ~netkit_features_hw_vlan; 269 270 dev->needs_free_netdev = true; 271 272 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 273 } 274 275 static struct net *netkit_get_link_net(const struct net_device *dev) 276 { 277 struct netkit *nk = netkit_priv(dev); 278 struct net_device *peer = rtnl_dereference(nk->peer); 279 280 return peer ? dev_net(peer) : dev_net(dev); 281 } 282 283 static int netkit_check_policy(int policy, struct nlattr *tb, 284 struct netlink_ext_ack *extack) 285 { 286 switch (policy) { 287 case NETKIT_PASS: 288 case NETKIT_DROP: 289 return 0; 290 default: 291 NL_SET_ERR_MSG_ATTR(extack, tb, 292 "Provided default xmit policy not supported"); 293 return -EINVAL; 294 } 295 } 296 297 static int netkit_check_mode(int mode, struct nlattr *tb, 298 struct netlink_ext_ack *extack) 299 { 300 switch (mode) { 301 case NETKIT_L2: 302 case NETKIT_L3: 303 return 0; 304 default: 305 NL_SET_ERR_MSG_ATTR(extack, tb, 306 "Provided device mode can only be L2 or L3"); 307 return -EINVAL; 308 } 309 } 310 311 static int netkit_validate(struct nlattr *tb[], struct nlattr *data[], 312 struct netlink_ext_ack *extack) 313 { 314 struct nlattr *attr = tb[IFLA_ADDRESS]; 315 316 if (!attr) 317 return 0; 318 if (nla_len(attr) != ETH_ALEN) 319 return -EINVAL; 320 if (!is_valid_ether_addr(nla_data(attr))) 321 return -EADDRNOTAVAIL; 322 return 0; 323 } 324 325 static struct rtnl_link_ops netkit_link_ops; 326 327 static int netkit_new_link(struct net *src_net, struct net_device *dev, 328 struct nlattr *tb[], struct nlattr *data[], 329 struct netlink_ext_ack *extack) 330 { 331 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp = tb, *attr; 332 enum netkit_action default_prim = NETKIT_PASS; 333 enum netkit_action default_peer = NETKIT_PASS; 334 enum netkit_mode mode = NETKIT_L3; 335 unsigned char ifname_assign_type; 336 struct ifinfomsg *ifmp = NULL; 337 struct net_device *peer; 338 char ifname[IFNAMSIZ]; 339 struct netkit *nk; 340 struct net *net; 341 int err; 342 343 if (data) { 344 if (data[IFLA_NETKIT_MODE]) { 345 attr = data[IFLA_NETKIT_MODE]; 346 mode = nla_get_u32(attr); 347 err = netkit_check_mode(mode, attr, extack); 348 if (err < 0) 349 return err; 350 } 351 if (data[IFLA_NETKIT_PEER_INFO]) { 352 attr = data[IFLA_NETKIT_PEER_INFO]; 353 ifmp = nla_data(attr); 354 err = rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack); 355 if (err < 0) 356 return err; 357 err = netkit_validate(peer_tb, NULL, extack); 358 if (err < 0) 359 return err; 360 tbp = peer_tb; 361 } 362 if (data[IFLA_NETKIT_POLICY]) { 363 attr = data[IFLA_NETKIT_POLICY]; 364 default_prim = nla_get_u32(attr); 365 err = netkit_check_policy(default_prim, attr, extack); 366 if (err < 0) 367 return err; 368 } 369 if (data[IFLA_NETKIT_PEER_POLICY]) { 370 attr = data[IFLA_NETKIT_PEER_POLICY]; 371 default_peer = nla_get_u32(attr); 372 err = netkit_check_policy(default_peer, attr, extack); 373 if (err < 0) 374 return err; 375 } 376 } 377 378 if (ifmp && tbp[IFLA_IFNAME]) { 379 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 380 ifname_assign_type = NET_NAME_USER; 381 } else { 382 strscpy(ifname, "nk%d", IFNAMSIZ); 383 ifname_assign_type = NET_NAME_ENUM; 384 } 385 if (mode != NETKIT_L2 && 386 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS])) 387 return -EOPNOTSUPP; 388 389 net = rtnl_link_get_net(src_net, tbp); 390 if (IS_ERR(net)) 391 return PTR_ERR(net); 392 393 peer = rtnl_create_link(net, ifname, ifname_assign_type, 394 &netkit_link_ops, tbp, extack); 395 if (IS_ERR(peer)) { 396 put_net(net); 397 return PTR_ERR(peer); 398 } 399 400 netif_inherit_tso_max(peer, dev); 401 402 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS])) 403 eth_hw_addr_random(peer); 404 if (ifmp && dev->ifindex) 405 peer->ifindex = ifmp->ifi_index; 406 407 nk = netkit_priv(peer); 408 nk->primary = false; 409 nk->policy = default_peer; 410 nk->mode = mode; 411 bpf_mprog_bundle_init(&nk->bundle); 412 413 err = register_netdevice(peer); 414 put_net(net); 415 if (err < 0) 416 goto err_register_peer; 417 netif_carrier_off(peer); 418 if (mode == NETKIT_L2) 419 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL); 420 421 err = rtnl_configure_link(peer, NULL, 0, NULL); 422 if (err < 0) 423 goto err_configure_peer; 424 425 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS]) 426 eth_hw_addr_random(dev); 427 if (tb[IFLA_IFNAME]) 428 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 429 else 430 strscpy(dev->name, "nk%d", IFNAMSIZ); 431 432 nk = netkit_priv(dev); 433 nk->primary = true; 434 nk->policy = default_prim; 435 nk->mode = mode; 436 bpf_mprog_bundle_init(&nk->bundle); 437 438 err = register_netdevice(dev); 439 if (err < 0) 440 goto err_configure_peer; 441 netif_carrier_off(dev); 442 if (mode == NETKIT_L2) 443 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL); 444 445 rcu_assign_pointer(netkit_priv(dev)->peer, peer); 446 rcu_assign_pointer(netkit_priv(peer)->peer, dev); 447 return 0; 448 err_configure_peer: 449 unregister_netdevice(peer); 450 return err; 451 err_register_peer: 452 free_netdev(peer); 453 return err; 454 } 455 456 static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev, 457 bool bundle_fallback) 458 { 459 struct netkit *nk = netkit_priv(dev); 460 struct bpf_mprog_entry *entry; 461 462 ASSERT_RTNL(); 463 entry = rcu_dereference_rtnl(nk->active); 464 if (entry) 465 return entry; 466 if (bundle_fallback) 467 return &nk->bundle.a; 468 return NULL; 469 } 470 471 static void netkit_entry_update(struct net_device *dev, 472 struct bpf_mprog_entry *entry) 473 { 474 struct netkit *nk = netkit_priv(dev); 475 476 ASSERT_RTNL(); 477 rcu_assign_pointer(nk->active, entry); 478 } 479 480 static void netkit_entry_sync(void) 481 { 482 synchronize_rcu(); 483 } 484 485 static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which) 486 { 487 struct net_device *dev; 488 struct netkit *nk; 489 490 ASSERT_RTNL(); 491 492 switch (which) { 493 case BPF_NETKIT_PRIMARY: 494 case BPF_NETKIT_PEER: 495 break; 496 default: 497 return ERR_PTR(-EINVAL); 498 } 499 500 dev = __dev_get_by_index(net, ifindex); 501 if (!dev) 502 return ERR_PTR(-ENODEV); 503 if (dev->netdev_ops != &netkit_netdev_ops) 504 return ERR_PTR(-ENXIO); 505 506 nk = netkit_priv(dev); 507 if (!nk->primary) 508 return ERR_PTR(-EACCES); 509 if (which == BPF_NETKIT_PEER) { 510 dev = rcu_dereference_rtnl(nk->peer); 511 if (!dev) 512 return ERR_PTR(-ENODEV); 513 } 514 return dev; 515 } 516 517 int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) 518 { 519 struct bpf_mprog_entry *entry, *entry_new; 520 struct bpf_prog *replace_prog = NULL; 521 struct net_device *dev; 522 int ret; 523 524 rtnl_lock(); 525 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 526 attr->attach_type); 527 if (IS_ERR(dev)) { 528 ret = PTR_ERR(dev); 529 goto out; 530 } 531 entry = netkit_entry_fetch(dev, true); 532 if (attr->attach_flags & BPF_F_REPLACE) { 533 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, 534 prog->type); 535 if (IS_ERR(replace_prog)) { 536 ret = PTR_ERR(replace_prog); 537 replace_prog = NULL; 538 goto out; 539 } 540 } 541 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, 542 attr->attach_flags, attr->relative_fd, 543 attr->expected_revision); 544 if (!ret) { 545 if (entry != entry_new) { 546 netkit_entry_update(dev, entry_new); 547 netkit_entry_sync(); 548 } 549 bpf_mprog_commit(entry); 550 } 551 out: 552 if (replace_prog) 553 bpf_prog_put(replace_prog); 554 rtnl_unlock(); 555 return ret; 556 } 557 558 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) 559 { 560 struct bpf_mprog_entry *entry, *entry_new; 561 struct net_device *dev; 562 int ret; 563 564 rtnl_lock(); 565 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 566 attr->attach_type); 567 if (IS_ERR(dev)) { 568 ret = PTR_ERR(dev); 569 goto out; 570 } 571 entry = netkit_entry_fetch(dev, false); 572 if (!entry) { 573 ret = -ENOENT; 574 goto out; 575 } 576 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, 577 attr->relative_fd, attr->expected_revision); 578 if (!ret) { 579 if (!bpf_mprog_total(entry_new)) 580 entry_new = NULL; 581 netkit_entry_update(dev, entry_new); 582 netkit_entry_sync(); 583 bpf_mprog_commit(entry); 584 } 585 out: 586 rtnl_unlock(); 587 return ret; 588 } 589 590 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) 591 { 592 struct net_device *dev; 593 int ret; 594 595 rtnl_lock(); 596 dev = netkit_dev_fetch(current->nsproxy->net_ns, 597 attr->query.target_ifindex, 598 attr->query.attach_type); 599 if (IS_ERR(dev)) { 600 ret = PTR_ERR(dev); 601 goto out; 602 } 603 ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false)); 604 out: 605 rtnl_unlock(); 606 return ret; 607 } 608 609 static struct netkit_link *netkit_link(const struct bpf_link *link) 610 { 611 return container_of(link, struct netkit_link, link); 612 } 613 614 static int netkit_link_prog_attach(struct bpf_link *link, u32 flags, 615 u32 id_or_fd, u64 revision) 616 { 617 struct netkit_link *nkl = netkit_link(link); 618 struct bpf_mprog_entry *entry, *entry_new; 619 struct net_device *dev = nkl->dev; 620 int ret; 621 622 ASSERT_RTNL(); 623 entry = netkit_entry_fetch(dev, true); 624 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, 625 id_or_fd, revision); 626 if (!ret) { 627 if (entry != entry_new) { 628 netkit_entry_update(dev, entry_new); 629 netkit_entry_sync(); 630 } 631 bpf_mprog_commit(entry); 632 } 633 return ret; 634 } 635 636 static void netkit_link_release(struct bpf_link *link) 637 { 638 struct netkit_link *nkl = netkit_link(link); 639 struct bpf_mprog_entry *entry, *entry_new; 640 struct net_device *dev; 641 int ret = 0; 642 643 rtnl_lock(); 644 dev = nkl->dev; 645 if (!dev) 646 goto out; 647 entry = netkit_entry_fetch(dev, false); 648 if (!entry) { 649 ret = -ENOENT; 650 goto out; 651 } 652 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); 653 if (!ret) { 654 if (!bpf_mprog_total(entry_new)) 655 entry_new = NULL; 656 netkit_entry_update(dev, entry_new); 657 netkit_entry_sync(); 658 bpf_mprog_commit(entry); 659 nkl->dev = NULL; 660 } 661 out: 662 WARN_ON_ONCE(ret); 663 rtnl_unlock(); 664 } 665 666 static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog, 667 struct bpf_prog *oprog) 668 { 669 struct netkit_link *nkl = netkit_link(link); 670 struct bpf_mprog_entry *entry, *entry_new; 671 struct net_device *dev; 672 int ret = 0; 673 674 rtnl_lock(); 675 dev = nkl->dev; 676 if (!dev) { 677 ret = -ENOLINK; 678 goto out; 679 } 680 if (oprog && link->prog != oprog) { 681 ret = -EPERM; 682 goto out; 683 } 684 oprog = link->prog; 685 if (oprog == nprog) { 686 bpf_prog_put(nprog); 687 goto out; 688 } 689 entry = netkit_entry_fetch(dev, false); 690 if (!entry) { 691 ret = -ENOENT; 692 goto out; 693 } 694 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, 695 BPF_F_REPLACE | BPF_F_ID, 696 link->prog->aux->id, 0); 697 if (!ret) { 698 WARN_ON_ONCE(entry != entry_new); 699 oprog = xchg(&link->prog, nprog); 700 bpf_prog_put(oprog); 701 bpf_mprog_commit(entry); 702 } 703 out: 704 rtnl_unlock(); 705 return ret; 706 } 707 708 static void netkit_link_dealloc(struct bpf_link *link) 709 { 710 kfree(netkit_link(link)); 711 } 712 713 static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) 714 { 715 const struct netkit_link *nkl = netkit_link(link); 716 u32 ifindex = 0; 717 718 rtnl_lock(); 719 if (nkl->dev) 720 ifindex = nkl->dev->ifindex; 721 rtnl_unlock(); 722 723 seq_printf(seq, "ifindex:\t%u\n", ifindex); 724 seq_printf(seq, "attach_type:\t%u (%s)\n", 725 nkl->location, 726 nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer"); 727 } 728 729 static int netkit_link_fill_info(const struct bpf_link *link, 730 struct bpf_link_info *info) 731 { 732 const struct netkit_link *nkl = netkit_link(link); 733 u32 ifindex = 0; 734 735 rtnl_lock(); 736 if (nkl->dev) 737 ifindex = nkl->dev->ifindex; 738 rtnl_unlock(); 739 740 info->netkit.ifindex = ifindex; 741 info->netkit.attach_type = nkl->location; 742 return 0; 743 } 744 745 static int netkit_link_detach(struct bpf_link *link) 746 { 747 netkit_link_release(link); 748 return 0; 749 } 750 751 static const struct bpf_link_ops netkit_link_lops = { 752 .release = netkit_link_release, 753 .detach = netkit_link_detach, 754 .dealloc = netkit_link_dealloc, 755 .update_prog = netkit_link_update, 756 .show_fdinfo = netkit_link_fdinfo, 757 .fill_link_info = netkit_link_fill_info, 758 }; 759 760 static int netkit_link_init(struct netkit_link *nkl, 761 struct bpf_link_primer *link_primer, 762 const union bpf_attr *attr, 763 struct net_device *dev, 764 struct bpf_prog *prog) 765 { 766 bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT, 767 &netkit_link_lops, prog); 768 nkl->location = attr->link_create.attach_type; 769 nkl->dev = dev; 770 return bpf_link_prime(&nkl->link, link_primer); 771 } 772 773 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 774 { 775 struct bpf_link_primer link_primer; 776 struct netkit_link *nkl; 777 struct net_device *dev; 778 int ret; 779 780 rtnl_lock(); 781 dev = netkit_dev_fetch(current->nsproxy->net_ns, 782 attr->link_create.target_ifindex, 783 attr->link_create.attach_type); 784 if (IS_ERR(dev)) { 785 ret = PTR_ERR(dev); 786 goto out; 787 } 788 nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT); 789 if (!nkl) { 790 ret = -ENOMEM; 791 goto out; 792 } 793 ret = netkit_link_init(nkl, &link_primer, attr, dev, prog); 794 if (ret) { 795 kfree(nkl); 796 goto out; 797 } 798 ret = netkit_link_prog_attach(&nkl->link, 799 attr->link_create.flags, 800 attr->link_create.netkit.relative_fd, 801 attr->link_create.netkit.expected_revision); 802 if (ret) { 803 nkl->dev = NULL; 804 bpf_link_cleanup(&link_primer); 805 goto out; 806 } 807 ret = bpf_link_settle(&link_primer); 808 out: 809 rtnl_unlock(); 810 return ret; 811 } 812 813 static void netkit_release_all(struct net_device *dev) 814 { 815 struct bpf_mprog_entry *entry; 816 struct bpf_tuple tuple = {}; 817 struct bpf_mprog_fp *fp; 818 struct bpf_mprog_cp *cp; 819 820 entry = netkit_entry_fetch(dev, false); 821 if (!entry) 822 return; 823 netkit_entry_update(dev, NULL); 824 netkit_entry_sync(); 825 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { 826 if (tuple.link) 827 netkit_link(tuple.link)->dev = NULL; 828 else 829 bpf_prog_put(tuple.prog); 830 } 831 } 832 833 static void netkit_uninit(struct net_device *dev) 834 { 835 netkit_release_all(dev); 836 } 837 838 static void netkit_del_link(struct net_device *dev, struct list_head *head) 839 { 840 struct netkit *nk = netkit_priv(dev); 841 struct net_device *peer = rtnl_dereference(nk->peer); 842 843 RCU_INIT_POINTER(nk->peer, NULL); 844 unregister_netdevice_queue(dev, head); 845 if (peer) { 846 nk = netkit_priv(peer); 847 RCU_INIT_POINTER(nk->peer, NULL); 848 unregister_netdevice_queue(peer, head); 849 } 850 } 851 852 static int netkit_change_link(struct net_device *dev, struct nlattr *tb[], 853 struct nlattr *data[], 854 struct netlink_ext_ack *extack) 855 { 856 struct netkit *nk = netkit_priv(dev); 857 struct net_device *peer = rtnl_dereference(nk->peer); 858 enum netkit_action policy; 859 struct nlattr *attr; 860 int err; 861 862 if (!nk->primary) { 863 NL_SET_ERR_MSG(extack, 864 "netkit link settings can be changed only through the primary device"); 865 return -EACCES; 866 } 867 868 if (data[IFLA_NETKIT_MODE]) { 869 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_MODE], 870 "netkit link operating mode cannot be changed after device creation"); 871 return -EACCES; 872 } 873 874 if (data[IFLA_NETKIT_PEER_INFO]) { 875 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_NETKIT_PEER_INFO], 876 "netkit peer info cannot be changed after device creation"); 877 return -EINVAL; 878 } 879 880 if (data[IFLA_NETKIT_POLICY]) { 881 attr = data[IFLA_NETKIT_POLICY]; 882 policy = nla_get_u32(attr); 883 err = netkit_check_policy(policy, attr, extack); 884 if (err) 885 return err; 886 WRITE_ONCE(nk->policy, policy); 887 } 888 889 if (data[IFLA_NETKIT_PEER_POLICY]) { 890 err = -EOPNOTSUPP; 891 attr = data[IFLA_NETKIT_PEER_POLICY]; 892 policy = nla_get_u32(attr); 893 if (peer) 894 err = netkit_check_policy(policy, attr, extack); 895 if (err) 896 return err; 897 nk = netkit_priv(peer); 898 WRITE_ONCE(nk->policy, policy); 899 } 900 901 return 0; 902 } 903 904 static size_t netkit_get_size(const struct net_device *dev) 905 { 906 return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */ 907 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */ 908 nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */ 909 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */ 910 0; 911 } 912 913 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev) 914 { 915 struct netkit *nk = netkit_priv(dev); 916 struct net_device *peer = rtnl_dereference(nk->peer); 917 918 if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary)) 919 return -EMSGSIZE; 920 if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy)) 921 return -EMSGSIZE; 922 if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode)) 923 return -EMSGSIZE; 924 925 if (peer) { 926 nk = netkit_priv(peer); 927 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy)) 928 return -EMSGSIZE; 929 } 930 931 return 0; 932 } 933 934 static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = { 935 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) }, 936 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 }, 937 [IFLA_NETKIT_MODE] = { .type = NLA_U32 }, 938 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 }, 939 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT, 940 .reject_message = "Primary attribute is read-only" }, 941 }; 942 943 static struct rtnl_link_ops netkit_link_ops = { 944 .kind = DRV_NAME, 945 .priv_size = sizeof(struct netkit), 946 .setup = netkit_setup, 947 .newlink = netkit_new_link, 948 .dellink = netkit_del_link, 949 .changelink = netkit_change_link, 950 .get_link_net = netkit_get_link_net, 951 .get_size = netkit_get_size, 952 .fill_info = netkit_fill_info, 953 .policy = netkit_policy, 954 .validate = netkit_validate, 955 .maxtype = IFLA_NETKIT_MAX, 956 }; 957 958 static __init int netkit_init(void) 959 { 960 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT || 961 (int)NETKIT_PASS != (int)TCX_PASS || 962 (int)NETKIT_DROP != (int)TCX_DROP || 963 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT); 964 965 return rtnl_link_register(&netkit_link_ops); 966 } 967 968 static __exit void netkit_exit(void) 969 { 970 rtnl_link_unregister(&netkit_link_ops); 971 } 972 973 module_init(netkit_init); 974 module_exit(netkit_exit); 975 976 MODULE_DESCRIPTION("BPF-programmable network device"); 977 MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>"); 978 MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>"); 979 MODULE_LICENSE("GPL"); 980 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 981