1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2023 Isovalent */ 3 4 #include <linux/netdevice.h> 5 #include <linux/ethtool.h> 6 #include <linux/etherdevice.h> 7 #include <linux/filter.h> 8 #include <linux/netfilter_netdev.h> 9 #include <linux/bpf_mprog.h> 10 #include <linux/indirect_call_wrapper.h> 11 12 #include <net/netdev_lock.h> 13 #include <net/netdev_queues.h> 14 #include <net/netdev_rx_queue.h> 15 #include <net/xdp_sock_drv.h> 16 #include <net/netkit.h> 17 #include <net/dst.h> 18 #include <net/tcx.h> 19 20 #define NETKIT_DRV_NAME "netkit" 21 22 #define NETKIT_NUM_RX_QUEUES_MAX 1024 23 #define NETKIT_NUM_TX_QUEUES_MAX 1 24 25 #define NETKIT_NUM_RX_QUEUES_REAL 1 26 #define NETKIT_NUM_TX_QUEUES_REAL 1 27 28 struct netkit { 29 __cacheline_group_begin(netkit_fastpath); 30 struct net_device __rcu *peer; 31 struct bpf_mprog_entry __rcu *active; 32 enum netkit_action policy; 33 enum netkit_scrub scrub; 34 struct bpf_mprog_bundle bundle; 35 __cacheline_group_end(netkit_fastpath); 36 37 __cacheline_group_begin(netkit_slowpath); 38 enum netkit_mode mode; 39 enum netkit_pairing pair; 40 bool primary; 41 u32 headroom; 42 __cacheline_group_end(netkit_slowpath); 43 }; 44 45 struct netkit_link { 46 struct bpf_link link; 47 struct net_device *dev; 48 }; 49 50 static struct rtnl_link_ops netkit_link_ops; 51 52 static __always_inline int 53 netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, 54 enum netkit_action ret) 55 { 56 const struct bpf_mprog_fp *fp; 57 const struct bpf_prog *prog; 58 59 bpf_mprog_foreach_prog(entry, fp, prog) { 60 bpf_compute_data_pointers(skb); 61 ret = bpf_prog_run(prog, skb); 62 if (ret != NETKIT_NEXT) 63 break; 64 } 65 return ret; 66 } 67 68 static void netkit_xnet(struct sk_buff *skb) 69 { 70 skb->priority = 0; 71 skb->mark = 0; 72 } 73 74 static void netkit_prep_forward(struct sk_buff *skb, 75 bool xnet, bool xnet_scrub) 76 { 77 skb_scrub_packet(skb, false); 78 nf_skip_egress(skb, true); 79 skb_reset_mac_header(skb); 80 if (!xnet) 81 return; 82 skb_clear_tstamp(skb); 83 if (xnet_scrub) 84 netkit_xnet(skb); 85 } 86 87 static struct netkit *netkit_priv(const struct net_device *dev) 88 { 89 return netdev_priv(dev); 90 } 91 92 static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev) 93 { 94 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 95 struct netkit *nk = netkit_priv(dev); 96 enum netkit_action ret = READ_ONCE(nk->policy); 97 netdev_tx_t ret_dev = NET_XMIT_SUCCESS; 98 const struct bpf_mprog_entry *entry; 99 struct net_device *peer; 100 int len = skb->len; 101 102 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 103 rcu_read_lock(); 104 peer = rcu_dereference(nk->peer); 105 if (unlikely(!peer || !(peer->flags & IFF_UP) || 106 !pskb_may_pull(skb, ETH_HLEN) || 107 skb_orphan_frags(skb, GFP_ATOMIC))) 108 goto drop; 109 netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), 110 nk->scrub); 111 eth_skb_pkt_type(skb, peer); 112 skb->dev = peer; 113 entry = rcu_dereference(nk->active); 114 if (entry) 115 ret = netkit_run(entry, skb, ret); 116 switch (ret) { 117 case NETKIT_NEXT: 118 case NETKIT_PASS: 119 eth_skb_pull_mac(skb); 120 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 121 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { 122 dev_sw_netstats_tx_add(dev, 1, len); 123 dev_sw_netstats_rx_add(peer, len); 124 } else { 125 goto drop_stats; 126 } 127 break; 128 case NETKIT_REDIRECT: 129 dev_sw_netstats_tx_add(dev, 1, len); 130 skb_do_redirect(skb); 131 break; 132 case NETKIT_DROP: 133 default: 134 drop: 135 kfree_skb(skb); 136 drop_stats: 137 dev_core_stats_tx_dropped_inc(dev); 138 ret_dev = NET_XMIT_DROP; 139 break; 140 } 141 rcu_read_unlock(); 142 bpf_net_ctx_clear(bpf_net_ctx); 143 return ret_dev; 144 } 145 146 static int netkit_open(struct net_device *dev) 147 { 148 struct netkit *nk = netkit_priv(dev); 149 struct net_device *peer = rtnl_dereference(nk->peer); 150 151 if (nk->pair == NETKIT_DEVICE_SINGLE) { 152 netif_carrier_on(dev); 153 return 0; 154 } 155 if (!peer) 156 return -ENOTCONN; 157 if (peer->flags & IFF_UP) { 158 netif_carrier_on(dev); 159 netif_carrier_on(peer); 160 } 161 return 0; 162 } 163 164 static int netkit_close(struct net_device *dev) 165 { 166 struct netkit *nk = netkit_priv(dev); 167 struct net_device *peer = rtnl_dereference(nk->peer); 168 169 netif_carrier_off(dev); 170 if (peer) 171 netif_carrier_off(peer); 172 return 0; 173 } 174 175 static int netkit_get_iflink(const struct net_device *dev) 176 { 177 struct netkit *nk = netkit_priv(dev); 178 struct net_device *peer; 179 int iflink = 0; 180 181 rcu_read_lock(); 182 peer = rcu_dereference(nk->peer); 183 if (peer) 184 iflink = READ_ONCE(peer->ifindex); 185 rcu_read_unlock(); 186 return iflink; 187 } 188 189 static void netkit_set_multicast(struct net_device *dev, 190 struct netdev_hw_addr_list *uc, 191 struct netdev_hw_addr_list *mc) 192 { 193 /* Nothing to do, we receive whatever gets pushed to us! */ 194 } 195 196 static int netkit_set_macaddr(struct net_device *dev, void *sa) 197 { 198 struct netkit *nk = netkit_priv(dev); 199 200 if (nk->mode != NETKIT_L2) 201 return -EOPNOTSUPP; 202 203 return eth_mac_addr(dev, sa); 204 } 205 206 static void netkit_set_headroom(struct net_device *dev, int headroom) 207 { 208 struct netkit *nk = netkit_priv(dev), *nk2; 209 struct net_device *peer; 210 211 if (headroom < 0) 212 headroom = NET_SKB_PAD; 213 214 rcu_read_lock(); 215 peer = rcu_dereference(nk->peer); 216 if (!peer) { 217 nk->headroom = headroom; 218 dev->needed_headroom = headroom; 219 } else { 220 nk2 = netkit_priv(peer); 221 nk->headroom = headroom; 222 headroom = max(nk->headroom, nk2->headroom); 223 224 peer->needed_headroom = headroom; 225 dev->needed_headroom = headroom; 226 } 227 rcu_read_unlock(); 228 } 229 230 INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev) 231 { 232 return rcu_dereference(netkit_priv(dev)->peer); 233 } 234 235 static void netkit_get_stats(struct net_device *dev, 236 struct rtnl_link_stats64 *stats) 237 { 238 dev_fetch_sw_netstats(stats, dev->tstats); 239 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 240 } 241 242 static bool netkit_xsk_supported_at_phys(const struct net_device *dev) 243 { 244 if (!dev->netdev_ops->ndo_bpf || 245 !dev->netdev_ops->ndo_xdp_xmit || 246 !dev->netdev_ops->ndo_xsk_wakeup) 247 return false; 248 return true; 249 } 250 251 static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp) 252 { 253 struct netkit *nk = netkit_priv(dev); 254 struct netdev_bpf xdp_lower; 255 struct netdev_rx_queue *rxq; 256 struct net_device *phys; 257 bool create = false; 258 int ret = -EBUSY; 259 260 switch (xdp->command) { 261 case XDP_SETUP_XSK_POOL: 262 if (nk->pair == NETKIT_DEVICE_PAIR) 263 return -EOPNOTSUPP; 264 if (xdp->xsk.queue_id >= dev->real_num_rx_queues) 265 return -EINVAL; 266 267 rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id); 268 if (!rxq->lease) 269 return -EOPNOTSUPP; 270 271 phys = rxq->lease->dev; 272 if (!netkit_xsk_supported_at_phys(phys)) 273 return -EOPNOTSUPP; 274 275 create = xdp->xsk.pool; 276 memcpy(&xdp_lower, xdp, sizeof(xdp_lower)); 277 xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease); 278 break; 279 case XDP_SETUP_PROG: 280 return -EOPNOTSUPP; 281 default: 282 return -EINVAL; 283 } 284 285 netdev_lock(phys); 286 if (create && 287 (phys->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) { 288 ret = -EOPNOTSUPP; 289 goto out; 290 } 291 if (!create || !dev_get_min_mp_channel_count(phys)) 292 ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower); 293 out: 294 netdev_unlock(phys); 295 return ret; 296 } 297 298 static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 299 { 300 struct netdev_rx_queue *rxq, *rxq_lease; 301 struct net_device *phys; 302 303 if (queue_id >= dev->real_num_rx_queues) 304 return -EINVAL; 305 306 rxq = __netif_get_rx_queue(dev, queue_id); 307 rxq_lease = READ_ONCE(rxq->lease); 308 if (unlikely(!rxq_lease)) 309 return -EOPNOTSUPP; 310 311 /* netkit_xsk already validated full xsk support, hence it's 312 * fine to call into ndo_xsk_wakeup right away given this 313 * was a prerequisite to get here in the first place. The 314 * phys xsk support cannot change without tearing down the 315 * device (which clears the lease first). 316 */ 317 phys = rxq_lease->dev; 318 return phys->netdev_ops->ndo_xsk_wakeup(phys, 319 get_netdev_rx_queue_index(rxq_lease), flags); 320 } 321 322 static int netkit_init(struct net_device *dev) 323 { 324 netdev_lockdep_set_classes(dev); 325 return 0; 326 } 327 328 static void netkit_uninit(struct net_device *dev); 329 330 static const struct net_device_ops netkit_netdev_ops = { 331 .ndo_init = netkit_init, 332 .ndo_open = netkit_open, 333 .ndo_stop = netkit_close, 334 .ndo_start_xmit = netkit_xmit, 335 .ndo_set_rx_mode_async = netkit_set_multicast, 336 .ndo_set_rx_headroom = netkit_set_headroom, 337 .ndo_set_mac_address = netkit_set_macaddr, 338 .ndo_get_iflink = netkit_get_iflink, 339 .ndo_get_peer_dev = netkit_peer_dev, 340 .ndo_get_stats64 = netkit_get_stats, 341 .ndo_uninit = netkit_uninit, 342 .ndo_bpf = netkit_xsk, 343 .ndo_xsk_wakeup = netkit_xsk_wakeup, 344 .ndo_features_check = passthru_features_check, 345 }; 346 347 static void netkit_get_drvinfo(struct net_device *dev, 348 struct ethtool_drvinfo *info) 349 { 350 strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver)); 351 } 352 353 static const struct ethtool_ops netkit_ethtool_ops = { 354 .get_drvinfo = netkit_get_drvinfo, 355 }; 356 357 static int netkit_queue_create(struct net_device *dev, 358 struct netlink_ext_ack *extack) 359 { 360 struct netkit *nk = netkit_priv(dev); 361 u32 rxq_count_old, rxq_count_new; 362 int err; 363 364 rxq_count_old = dev->real_num_rx_queues; 365 rxq_count_new = rxq_count_old + 1; 366 367 /* In paired mode, only the non-primary (peer) device can 368 * create leased queues since the primary is the management 369 * side. In single device mode, leasing is always allowed. 370 */ 371 if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) { 372 NL_SET_ERR_MSG(extack, 373 "netkit can only lease against the peer device"); 374 return -EOPNOTSUPP; 375 } 376 377 err = netif_set_real_num_rx_queues(dev, rxq_count_new); 378 if (err) { 379 if (rxq_count_new > dev->num_rx_queues) 380 NL_SET_ERR_MSG(extack, 381 "netkit maximum queue limit reached"); 382 else 383 NL_SET_ERR_MSG_FMT(extack, 384 "netkit cannot create more queues err=%d", err); 385 return err; 386 } 387 388 return rxq_count_old; 389 } 390 391 static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = { 392 .ndo_queue_create = netkit_queue_create, 393 }; 394 395 static struct net_device *netkit_alloc(struct nlattr *tb[], 396 const char *ifname, 397 unsigned char name_assign_type, 398 unsigned int num_tx_queues, 399 unsigned int num_rx_queues) 400 { 401 const struct rtnl_link_ops *ops = &netkit_link_ops; 402 struct net_device *dev; 403 404 if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX || 405 num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX) 406 return ERR_PTR(-EOPNOTSUPP); 407 408 dev = alloc_netdev_mqs(ops->priv_size, ifname, 409 name_assign_type, ops->setup, 410 num_tx_queues, num_rx_queues); 411 if (dev) { 412 dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL; 413 dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL; 414 } 415 return dev; 416 } 417 418 static void netkit_queue_unlease(struct net_device *dev) 419 { 420 struct netdev_rx_queue *rxq, *rxq_lease; 421 struct net_device *dev_lease; 422 int i; 423 424 if (dev->real_num_rx_queues == 1) 425 return; 426 427 netdev_lock(dev); 428 for (i = 1; i < dev->real_num_rx_queues; i++) { 429 rxq = __netif_get_rx_queue(dev, i); 430 rxq_lease = rxq->lease; 431 dev_lease = rxq_lease->dev; 432 433 netdev_lock(dev_lease); 434 netdev_rx_queue_unlease(rxq, rxq_lease); 435 netdev_unlock(dev_lease); 436 } 437 netdev_unlock(dev); 438 } 439 440 static void netkit_setup(struct net_device *dev) 441 { 442 static const netdev_features_t netkit_features_hw_vlan = 443 NETIF_F_HW_VLAN_CTAG_TX | 444 NETIF_F_HW_VLAN_CTAG_RX | 445 NETIF_F_HW_VLAN_STAG_TX | 446 NETIF_F_HW_VLAN_STAG_RX; 447 static const netdev_features_t netkit_features = 448 netkit_features_hw_vlan | 449 NETIF_F_SG | 450 NETIF_F_FRAGLIST | 451 NETIF_F_HW_CSUM | 452 NETIF_F_RXCSUM | 453 NETIF_F_SCTP_CRC | 454 NETIF_F_HIGHDMA | 455 NETIF_F_GSO_SOFTWARE | 456 NETIF_F_GSO_ENCAP_ALL; 457 458 ether_setup(dev); 459 dev->max_mtu = ETH_MAX_MTU; 460 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 461 462 dev->flags |= IFF_NOARP; 463 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 464 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 465 dev->priv_flags |= IFF_PHONY_HEADROOM; 466 dev->priv_flags |= IFF_NO_QUEUE; 467 dev->priv_flags |= IFF_DISABLE_NETPOLL; 468 dev->lltx = true; 469 dev->netmem_tx = NETMEM_TX_NO_DMA; 470 471 dev->netdev_ops = &netkit_netdev_ops; 472 dev->ethtool_ops = &netkit_ethtool_ops; 473 dev->queue_mgmt_ops = &netkit_queue_mgmt_ops; 474 475 dev->features |= netkit_features; 476 dev->hw_features = netkit_features; 477 dev->hw_enc_features = netkit_features; 478 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 479 dev->vlan_features = dev->features & ~netkit_features_hw_vlan; 480 481 dev->needs_free_netdev = true; 482 483 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 484 } 485 486 static struct net *netkit_get_link_net(const struct net_device *dev) 487 { 488 struct netkit *nk = netkit_priv(dev); 489 struct net_device *peer = rtnl_dereference(nk->peer); 490 491 return peer ? dev_net(peer) : dev_net(dev); 492 } 493 494 static int netkit_check_policy(int policy, struct nlattr *tb, 495 struct netlink_ext_ack *extack) 496 { 497 switch (policy) { 498 case NETKIT_PASS: 499 case NETKIT_DROP: 500 return 0; 501 default: 502 NL_SET_ERR_MSG_ATTR(extack, tb, 503 "Provided default xmit policy not supported"); 504 return -EINVAL; 505 } 506 } 507 508 static int netkit_validate(struct nlattr *tb[], struct nlattr *data[], 509 struct netlink_ext_ack *extack) 510 { 511 struct nlattr *attr = tb[IFLA_ADDRESS]; 512 513 if (!attr) 514 return 0; 515 if (nla_len(attr) != ETH_ALEN) 516 return -EINVAL; 517 if (!is_valid_ether_addr(nla_data(attr))) 518 return -EADDRNOTAVAIL; 519 return 0; 520 } 521 522 static int netkit_new_link(struct net_device *dev, 523 struct rtnl_newlink_params *params, 524 struct netlink_ext_ack *extack) 525 { 526 struct net *peer_net = rtnl_newlink_peer_net(params); 527 enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT; 528 enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT; 529 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr; 530 enum netkit_pairing pair = NETKIT_DEVICE_PAIR; 531 enum netkit_action policy_prim = NETKIT_PASS; 532 enum netkit_action policy_peer = NETKIT_PASS; 533 bool seen_peer = false, seen_scrub = false; 534 struct nlattr **data = params->data; 535 enum netkit_mode mode = NETKIT_L3; 536 unsigned char ifname_assign_type; 537 struct nlattr **tb = params->tb; 538 u16 headroom = 0, tailroom = 0; 539 struct ifinfomsg *ifmp = NULL; 540 struct net_device *peer = NULL; 541 char ifname[IFNAMSIZ]; 542 struct netkit *nk; 543 int err; 544 545 tbp = tb; 546 if (data) { 547 if (data[IFLA_NETKIT_MODE]) 548 mode = nla_get_u32(data[IFLA_NETKIT_MODE]); 549 if (data[IFLA_NETKIT_PEER_INFO]) { 550 attr = data[IFLA_NETKIT_PEER_INFO]; 551 ifmp = nla_data(attr); 552 rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack); 553 tbp = peer_tb; 554 } 555 if (data[IFLA_NETKIT_SCRUB]) 556 scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]); 557 if (data[IFLA_NETKIT_PEER_SCRUB]) 558 scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]); 559 if (data[IFLA_NETKIT_POLICY]) { 560 attr = data[IFLA_NETKIT_POLICY]; 561 policy_prim = nla_get_u32(attr); 562 err = netkit_check_policy(policy_prim, attr, extack); 563 if (err < 0) 564 return err; 565 } 566 if (data[IFLA_NETKIT_PEER_POLICY]) { 567 attr = data[IFLA_NETKIT_PEER_POLICY]; 568 policy_peer = nla_get_u32(attr); 569 err = netkit_check_policy(policy_peer, attr, extack); 570 if (err < 0) 571 return err; 572 } 573 if (data[IFLA_NETKIT_HEADROOM]) 574 headroom = nla_get_u16(data[IFLA_NETKIT_HEADROOM]); 575 if (data[IFLA_NETKIT_TAILROOM]) 576 tailroom = nla_get_u16(data[IFLA_NETKIT_TAILROOM]); 577 if (data[IFLA_NETKIT_PAIRING]) 578 pair = nla_get_u32(data[IFLA_NETKIT_PAIRING]); 579 580 seen_scrub = data[IFLA_NETKIT_SCRUB]; 581 seen_peer = data[IFLA_NETKIT_PEER_INFO] || 582 data[IFLA_NETKIT_PEER_SCRUB] || 583 data[IFLA_NETKIT_PEER_POLICY]; 584 } 585 586 if (ifmp && tbp[IFLA_IFNAME]) { 587 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 588 ifname_assign_type = NET_NAME_USER; 589 } else { 590 strscpy(ifname, "nk%d", IFNAMSIZ); 591 ifname_assign_type = NET_NAME_ENUM; 592 } 593 if (mode != NETKIT_L2 && 594 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS])) 595 return -EOPNOTSUPP; 596 if (pair == NETKIT_DEVICE_SINGLE && 597 (tb != tbp || seen_peer || seen_scrub || 598 policy_prim != NETKIT_PASS)) 599 return -EOPNOTSUPP; 600 601 if (pair == NETKIT_DEVICE_PAIR) { 602 peer = rtnl_create_link(peer_net, ifname, ifname_assign_type, 603 &netkit_link_ops, tbp, extack); 604 if (IS_ERR(peer)) 605 return PTR_ERR(peer); 606 607 netif_inherit_tso_max(peer, dev); 608 if (headroom) 609 peer->needed_headroom = headroom; 610 if (tailroom) 611 peer->needed_tailroom = tailroom; 612 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS])) 613 eth_hw_addr_random(peer); 614 if (ifmp && dev->ifindex) 615 peer->ifindex = ifmp->ifi_index; 616 617 nk = netkit_priv(peer); 618 nk->primary = false; 619 nk->policy = policy_peer; 620 nk->scrub = scrub_peer; 621 nk->mode = mode; 622 nk->pair = pair; 623 nk->headroom = headroom; 624 bpf_mprog_bundle_init(&nk->bundle); 625 626 err = register_netdevice(peer); 627 if (err < 0) 628 goto err_register_peer; 629 netif_carrier_off(peer); 630 if (mode == NETKIT_L2) 631 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL); 632 633 err = rtnl_configure_link(peer, NULL, 0, NULL); 634 if (err < 0) 635 goto err_configure_peer; 636 } 637 638 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS]) 639 eth_hw_addr_random(dev); 640 if (tb[IFLA_IFNAME]) 641 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 642 else 643 strscpy(dev->name, "nk%d", IFNAMSIZ); 644 if (headroom) 645 dev->needed_headroom = headroom; 646 if (tailroom) 647 dev->needed_tailroom = tailroom; 648 649 nk = netkit_priv(dev); 650 nk->primary = true; 651 nk->policy = policy_prim; 652 nk->scrub = scrub_prim; 653 nk->mode = mode; 654 nk->pair = pair; 655 nk->headroom = headroom; 656 bpf_mprog_bundle_init(&nk->bundle); 657 658 if (pair == NETKIT_DEVICE_SINGLE) 659 xdp_set_features_flag(dev, NETDEV_XDP_ACT_XSK); 660 661 err = register_netdevice(dev); 662 if (err < 0) 663 goto err_configure_peer; 664 netif_carrier_off(dev); 665 if (mode == NETKIT_L2) 666 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL); 667 668 rcu_assign_pointer(netkit_priv(dev)->peer, peer); 669 if (peer) 670 rcu_assign_pointer(netkit_priv(peer)->peer, dev); 671 return 0; 672 err_configure_peer: 673 if (peer) 674 unregister_netdevice(peer); 675 return err; 676 err_register_peer: 677 free_netdev(peer); 678 return err; 679 } 680 681 static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev, 682 bool bundle_fallback) 683 { 684 struct netkit *nk = netkit_priv(dev); 685 struct bpf_mprog_entry *entry; 686 687 ASSERT_RTNL(); 688 entry = rcu_dereference_rtnl(nk->active); 689 if (entry) 690 return entry; 691 if (bundle_fallback) 692 return &nk->bundle.a; 693 return NULL; 694 } 695 696 static void netkit_entry_update(struct net_device *dev, 697 struct bpf_mprog_entry *entry) 698 { 699 struct netkit *nk = netkit_priv(dev); 700 701 ASSERT_RTNL(); 702 rcu_assign_pointer(nk->active, entry); 703 } 704 705 static void netkit_entry_sync(void) 706 { 707 synchronize_rcu(); 708 } 709 710 static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which) 711 { 712 struct net_device *dev; 713 struct netkit *nk; 714 715 ASSERT_RTNL(); 716 717 switch (which) { 718 case BPF_NETKIT_PRIMARY: 719 case BPF_NETKIT_PEER: 720 break; 721 default: 722 return ERR_PTR(-EINVAL); 723 } 724 725 dev = __dev_get_by_index(net, ifindex); 726 if (!dev) 727 return ERR_PTR(-ENODEV); 728 if (dev->netdev_ops != &netkit_netdev_ops) 729 return ERR_PTR(-ENXIO); 730 731 nk = netkit_priv(dev); 732 if (!nk->primary) 733 return ERR_PTR(-EACCES); 734 if (nk->pair == NETKIT_DEVICE_SINGLE) 735 return ERR_PTR(-EOPNOTSUPP); 736 if (which == BPF_NETKIT_PEER) { 737 dev = rcu_dereference_rtnl(nk->peer); 738 if (!dev) 739 return ERR_PTR(-ENODEV); 740 } 741 return dev; 742 } 743 744 int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) 745 { 746 struct bpf_mprog_entry *entry, *entry_new; 747 struct bpf_prog *replace_prog = NULL; 748 struct net_device *dev; 749 int ret; 750 751 rtnl_lock(); 752 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 753 attr->attach_type); 754 if (IS_ERR(dev)) { 755 ret = PTR_ERR(dev); 756 goto out; 757 } 758 entry = netkit_entry_fetch(dev, true); 759 if (attr->attach_flags & BPF_F_REPLACE) { 760 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, 761 prog->type); 762 if (IS_ERR(replace_prog)) { 763 ret = PTR_ERR(replace_prog); 764 replace_prog = NULL; 765 goto out; 766 } 767 } 768 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, 769 attr->attach_flags, attr->relative_fd, 770 attr->expected_revision); 771 if (!ret) { 772 if (entry != entry_new) { 773 netkit_entry_update(dev, entry_new); 774 netkit_entry_sync(); 775 } 776 bpf_mprog_commit(entry); 777 } 778 out: 779 if (replace_prog) 780 bpf_prog_put(replace_prog); 781 rtnl_unlock(); 782 return ret; 783 } 784 785 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) 786 { 787 struct bpf_mprog_entry *entry, *entry_new; 788 struct net_device *dev; 789 int ret; 790 791 rtnl_lock(); 792 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 793 attr->attach_type); 794 if (IS_ERR(dev)) { 795 ret = PTR_ERR(dev); 796 goto out; 797 } 798 entry = netkit_entry_fetch(dev, false); 799 if (!entry) { 800 ret = -ENOENT; 801 goto out; 802 } 803 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, 804 attr->relative_fd, attr->expected_revision); 805 if (!ret) { 806 if (!bpf_mprog_total(entry_new)) 807 entry_new = NULL; 808 netkit_entry_update(dev, entry_new); 809 netkit_entry_sync(); 810 bpf_mprog_commit(entry); 811 } 812 out: 813 rtnl_unlock(); 814 return ret; 815 } 816 817 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) 818 { 819 struct net_device *dev; 820 int ret; 821 822 rtnl_lock(); 823 dev = netkit_dev_fetch(current->nsproxy->net_ns, 824 attr->query.target_ifindex, 825 attr->query.attach_type); 826 if (IS_ERR(dev)) { 827 ret = PTR_ERR(dev); 828 goto out; 829 } 830 ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false)); 831 out: 832 rtnl_unlock(); 833 return ret; 834 } 835 836 static struct netkit_link *netkit_link(const struct bpf_link *link) 837 { 838 return container_of(link, struct netkit_link, link); 839 } 840 841 static int netkit_link_prog_attach(struct bpf_link *link, u32 flags, 842 u32 id_or_fd, u64 revision) 843 { 844 struct netkit_link *nkl = netkit_link(link); 845 struct bpf_mprog_entry *entry, *entry_new; 846 struct net_device *dev = nkl->dev; 847 int ret; 848 849 ASSERT_RTNL(); 850 entry = netkit_entry_fetch(dev, true); 851 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, 852 id_or_fd, revision); 853 if (!ret) { 854 if (entry != entry_new) { 855 netkit_entry_update(dev, entry_new); 856 netkit_entry_sync(); 857 } 858 bpf_mprog_commit(entry); 859 } 860 return ret; 861 } 862 863 static void netkit_link_release(struct bpf_link *link) 864 { 865 struct netkit_link *nkl = netkit_link(link); 866 struct bpf_mprog_entry *entry, *entry_new; 867 struct net_device *dev; 868 int ret = 0; 869 870 rtnl_lock(); 871 dev = nkl->dev; 872 if (!dev) 873 goto out; 874 entry = netkit_entry_fetch(dev, false); 875 if (!entry) { 876 ret = -ENOENT; 877 goto out; 878 } 879 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); 880 if (!ret) { 881 if (!bpf_mprog_total(entry_new)) 882 entry_new = NULL; 883 netkit_entry_update(dev, entry_new); 884 netkit_entry_sync(); 885 bpf_mprog_commit(entry); 886 nkl->dev = NULL; 887 } 888 out: 889 WARN_ON_ONCE(ret); 890 rtnl_unlock(); 891 } 892 893 static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog, 894 struct bpf_prog *oprog) 895 { 896 struct netkit_link *nkl = netkit_link(link); 897 struct bpf_mprog_entry *entry, *entry_new; 898 struct net_device *dev; 899 int ret = 0; 900 901 rtnl_lock(); 902 dev = nkl->dev; 903 if (!dev) { 904 ret = -ENOLINK; 905 goto out; 906 } 907 if (oprog && link->prog != oprog) { 908 ret = -EPERM; 909 goto out; 910 } 911 oprog = link->prog; 912 if (oprog == nprog) { 913 bpf_prog_put(nprog); 914 goto out; 915 } 916 entry = netkit_entry_fetch(dev, false); 917 if (!entry) { 918 ret = -ENOENT; 919 goto out; 920 } 921 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, 922 BPF_F_REPLACE | BPF_F_ID, 923 link->prog->aux->id, 0); 924 if (!ret) { 925 WARN_ON_ONCE(entry != entry_new); 926 oprog = xchg(&link->prog, nprog); 927 bpf_prog_put(oprog); 928 bpf_mprog_commit(entry); 929 } 930 out: 931 rtnl_unlock(); 932 return ret; 933 } 934 935 static void netkit_link_dealloc(struct bpf_link *link) 936 { 937 kfree(netkit_link(link)); 938 } 939 940 static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) 941 { 942 const struct netkit_link *nkl = netkit_link(link); 943 u32 ifindex = 0; 944 945 rtnl_lock(); 946 if (nkl->dev) 947 ifindex = nkl->dev->ifindex; 948 rtnl_unlock(); 949 950 seq_printf(seq, "ifindex:\t%u\n", ifindex); 951 seq_printf(seq, "attach_type:\t%u (%s)\n", 952 link->attach_type, 953 link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer"); 954 } 955 956 static int netkit_link_fill_info(const struct bpf_link *link, 957 struct bpf_link_info *info) 958 { 959 const struct netkit_link *nkl = netkit_link(link); 960 u32 ifindex = 0; 961 962 rtnl_lock(); 963 if (nkl->dev) 964 ifindex = nkl->dev->ifindex; 965 rtnl_unlock(); 966 967 info->netkit.ifindex = ifindex; 968 info->netkit.attach_type = link->attach_type; 969 return 0; 970 } 971 972 static int netkit_link_detach(struct bpf_link *link) 973 { 974 netkit_link_release(link); 975 return 0; 976 } 977 978 static const struct bpf_link_ops netkit_link_lops = { 979 .release = netkit_link_release, 980 .detach = netkit_link_detach, 981 .dealloc = netkit_link_dealloc, 982 .update_prog = netkit_link_update, 983 .show_fdinfo = netkit_link_fdinfo, 984 .fill_link_info = netkit_link_fill_info, 985 }; 986 987 static int netkit_link_init(struct netkit_link *nkl, 988 struct bpf_link_primer *link_primer, 989 const union bpf_attr *attr, 990 struct net_device *dev, 991 struct bpf_prog *prog) 992 { 993 bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT, 994 &netkit_link_lops, prog, attr->link_create.attach_type); 995 nkl->dev = dev; 996 return bpf_link_prime(&nkl->link, link_primer); 997 } 998 999 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 1000 { 1001 struct bpf_link_primer link_primer; 1002 struct netkit_link *nkl; 1003 struct net_device *dev; 1004 int ret; 1005 1006 rtnl_lock(); 1007 dev = netkit_dev_fetch(current->nsproxy->net_ns, 1008 attr->link_create.target_ifindex, 1009 attr->link_create.attach_type); 1010 if (IS_ERR(dev)) { 1011 ret = PTR_ERR(dev); 1012 goto out; 1013 } 1014 nkl = kzalloc_obj(*nkl, GFP_KERNEL_ACCOUNT); 1015 if (!nkl) { 1016 ret = -ENOMEM; 1017 goto out; 1018 } 1019 ret = netkit_link_init(nkl, &link_primer, attr, dev, prog); 1020 if (ret) { 1021 kfree(nkl); 1022 goto out; 1023 } 1024 ret = netkit_link_prog_attach(&nkl->link, 1025 attr->link_create.flags, 1026 attr->link_create.netkit.relative_fd, 1027 attr->link_create.netkit.expected_revision); 1028 if (ret) { 1029 nkl->dev = NULL; 1030 bpf_link_cleanup(&link_primer); 1031 goto out; 1032 } 1033 ret = bpf_link_settle(&link_primer); 1034 out: 1035 rtnl_unlock(); 1036 return ret; 1037 } 1038 1039 static void netkit_release_all(struct net_device *dev) 1040 { 1041 struct bpf_mprog_entry *entry; 1042 struct bpf_tuple tuple = {}; 1043 struct bpf_mprog_fp *fp; 1044 struct bpf_mprog_cp *cp; 1045 1046 entry = netkit_entry_fetch(dev, false); 1047 if (!entry) 1048 return; 1049 netkit_entry_update(dev, NULL); 1050 netkit_entry_sync(); 1051 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { 1052 if (tuple.link) 1053 netkit_link(tuple.link)->dev = NULL; 1054 else 1055 bpf_prog_put(tuple.prog); 1056 } 1057 } 1058 1059 static void netkit_uninit(struct net_device *dev) 1060 { 1061 netkit_release_all(dev); 1062 netkit_queue_unlease(dev); 1063 } 1064 1065 static void netkit_del_link(struct net_device *dev, struct list_head *head) 1066 { 1067 struct netkit *nk = netkit_priv(dev); 1068 struct net_device *peer = rtnl_dereference(nk->peer); 1069 1070 RCU_INIT_POINTER(nk->peer, NULL); 1071 unregister_netdevice_queue(dev, head); 1072 if (peer) { 1073 nk = netkit_priv(peer); 1074 RCU_INIT_POINTER(nk->peer, NULL); 1075 /* Guard against the peer already being in an unregister 1076 * list (e.g. same-namespace teardown where the peer is 1077 * in the caller's dev_kill_list). list_move_tail() on an 1078 * already-queued device would otherwise corrupt that 1079 * list's iteration. This situation can occur via netkit 1080 * notifier, hence guard against this scenario. 1081 */ 1082 if (!unregister_netdevice_queued(peer)) 1083 unregister_netdevice_queue(peer, head); 1084 } 1085 } 1086 1087 static int netkit_change_link(struct net_device *dev, struct nlattr *tb[], 1088 struct nlattr *data[], 1089 struct netlink_ext_ack *extack) 1090 { 1091 struct netkit *nk = netkit_priv(dev); 1092 struct net_device *peer = rtnl_dereference(nk->peer); 1093 enum netkit_action policy; 1094 struct nlattr *attr; 1095 int err, i; 1096 static const struct { 1097 u32 attr; 1098 char *name; 1099 } fixed_params[] = { 1100 { IFLA_NETKIT_MODE, "operating mode" }, 1101 { IFLA_NETKIT_SCRUB, "scrubbing" }, 1102 { IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" }, 1103 { IFLA_NETKIT_PEER_INFO, "peer info" }, 1104 { IFLA_NETKIT_HEADROOM, "headroom" }, 1105 { IFLA_NETKIT_TAILROOM, "tailroom" }, 1106 { IFLA_NETKIT_PAIRING, "pairing" }, 1107 }; 1108 1109 if (!nk->primary) { 1110 NL_SET_ERR_MSG(extack, 1111 "netkit link settings can be changed only through the primary device"); 1112 return -EACCES; 1113 } 1114 1115 for (i = 0; i < ARRAY_SIZE(fixed_params); i++) { 1116 attr = data[fixed_params[i].attr]; 1117 if (attr) { 1118 NL_SET_ERR_MSG_ATTR_FMT(extack, attr, 1119 "netkit link %s cannot be changed after device creation", 1120 fixed_params[i].name); 1121 return -EACCES; 1122 } 1123 } 1124 1125 if (data[IFLA_NETKIT_POLICY]) { 1126 err = -EOPNOTSUPP; 1127 attr = data[IFLA_NETKIT_POLICY]; 1128 policy = nla_get_u32(attr); 1129 if (nk->pair == NETKIT_DEVICE_PAIR) 1130 err = netkit_check_policy(policy, attr, extack); 1131 if (err) 1132 return err; 1133 WRITE_ONCE(nk->policy, policy); 1134 } 1135 1136 if (data[IFLA_NETKIT_PEER_POLICY]) { 1137 err = -EOPNOTSUPP; 1138 attr = data[IFLA_NETKIT_PEER_POLICY]; 1139 policy = nla_get_u32(attr); 1140 if (peer) 1141 err = netkit_check_policy(policy, attr, extack); 1142 if (err) 1143 return err; 1144 nk = netkit_priv(peer); 1145 WRITE_ONCE(nk->policy, policy); 1146 } 1147 1148 return 0; 1149 } 1150 1151 static void netkit_check_lease_unregister(struct net_device *dev) 1152 { 1153 LIST_HEAD(list_kill); 1154 u32 q_idx; 1155 1156 if (READ_ONCE(dev->reg_state) != NETREG_UNREGISTERING || 1157 !dev->dev.parent) 1158 return; 1159 1160 netdev_lock_ops(dev); 1161 for (q_idx = 0; q_idx < dev->real_num_rx_queues; q_idx++) { 1162 struct net_device *tmp = dev; 1163 struct netdev_rx_queue *rxq; 1164 u32 tmp_q_idx = q_idx; 1165 1166 rxq = __netif_get_rx_queue_lease(&tmp, &tmp_q_idx, 1167 NETIF_PHYS_TO_VIRT); 1168 if (rxq && tmp != dev && 1169 tmp->netdev_ops == &netkit_netdev_ops) { 1170 /* A single phys device can have multiple queues leased 1171 * to one netkit device. We can only queue that netkit 1172 * device once to the list_kill. Queues of that phys 1173 * device can be leased with different individual netkit 1174 * devices, hence we batch via list_kill. 1175 */ 1176 if (unregister_netdevice_queued(tmp)) 1177 continue; 1178 netkit_del_link(tmp, &list_kill); 1179 } 1180 } 1181 netdev_unlock_ops(dev); 1182 unregister_netdevice_many(&list_kill); 1183 } 1184 1185 static int netkit_notifier(struct notifier_block *this, 1186 unsigned long event, void *ptr) 1187 { 1188 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 1189 1190 if (event == NETDEV_UNREGISTER) 1191 netkit_check_lease_unregister(dev); 1192 return NOTIFY_DONE; 1193 } 1194 1195 static size_t netkit_get_size(const struct net_device *dev) 1196 { 1197 return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */ 1198 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */ 1199 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */ 1200 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */ 1201 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */ 1202 nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */ 1203 nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */ 1204 nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */ 1205 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PAIRING */ 1206 0; 1207 } 1208 1209 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev) 1210 { 1211 struct netkit *nk = netkit_priv(dev); 1212 struct net_device *peer = rtnl_dereference(nk->peer); 1213 1214 if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary)) 1215 return -EMSGSIZE; 1216 if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy)) 1217 return -EMSGSIZE; 1218 if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode)) 1219 return -EMSGSIZE; 1220 if (nk->pair == NETKIT_DEVICE_PAIR && 1221 nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub)) 1222 return -EMSGSIZE; 1223 if (nla_put_u16(skb, IFLA_NETKIT_HEADROOM, dev->needed_headroom)) 1224 return -EMSGSIZE; 1225 if (nla_put_u16(skb, IFLA_NETKIT_TAILROOM, dev->needed_tailroom)) 1226 return -EMSGSIZE; 1227 if (nla_put_u32(skb, IFLA_NETKIT_PAIRING, nk->pair)) 1228 return -EMSGSIZE; 1229 1230 if (peer) { 1231 nk = netkit_priv(peer); 1232 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy)) 1233 return -EMSGSIZE; 1234 if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub)) 1235 return -EMSGSIZE; 1236 } 1237 1238 return 0; 1239 } 1240 1241 static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = { 1242 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) }, 1243 [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3), 1244 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 }, 1245 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 }, 1246 [IFLA_NETKIT_HEADROOM] = { .type = NLA_U16 }, 1247 [IFLA_NETKIT_TAILROOM] = { .type = NLA_U16 }, 1248 [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), 1249 [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), 1250 [IFLA_NETKIT_PAIRING] = NLA_POLICY_MAX(NLA_U32, NETKIT_DEVICE_SINGLE), 1251 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT, 1252 .reject_message = "Primary attribute is read-only" }, 1253 }; 1254 1255 static struct rtnl_link_ops netkit_link_ops = { 1256 .kind = NETKIT_DRV_NAME, 1257 .priv_size = sizeof(struct netkit), 1258 .alloc = netkit_alloc, 1259 .setup = netkit_setup, 1260 .newlink = netkit_new_link, 1261 .dellink = netkit_del_link, 1262 .changelink = netkit_change_link, 1263 .get_link_net = netkit_get_link_net, 1264 .get_size = netkit_get_size, 1265 .fill_info = netkit_fill_info, 1266 .policy = netkit_policy, 1267 .validate = netkit_validate, 1268 .peer_type = IFLA_NETKIT_PEER_INFO, 1269 .maxtype = IFLA_NETKIT_MAX, 1270 }; 1271 1272 static struct notifier_block netkit_netdev_notifier = { 1273 .notifier_call = netkit_notifier, 1274 }; 1275 1276 static __init int netkit_mod_init(void) 1277 { 1278 int ret; 1279 1280 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT || 1281 (int)NETKIT_PASS != (int)TCX_PASS || 1282 (int)NETKIT_DROP != (int)TCX_DROP || 1283 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT); 1284 1285 ret = rtnl_link_register(&netkit_link_ops); 1286 if (ret) 1287 return ret; 1288 ret = register_netdevice_notifier(&netkit_netdev_notifier); 1289 if (ret) 1290 rtnl_link_unregister(&netkit_link_ops); 1291 return ret; 1292 } 1293 1294 static __exit void netkit_mod_exit(void) 1295 { 1296 unregister_netdevice_notifier(&netkit_netdev_notifier); 1297 rtnl_link_unregister(&netkit_link_ops); 1298 } 1299 1300 module_init(netkit_mod_init); 1301 module_exit(netkit_mod_exit); 1302 1303 MODULE_DESCRIPTION("BPF-programmable network device"); 1304 MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>"); 1305 MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>"); 1306 MODULE_LICENSE("GPL"); 1307 MODULE_ALIAS_RTNL_LINK(NETKIT_DRV_NAME); 1308