1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 #include <net/busy_poll.h> 33 34 #include "netdevsim.h" 35 36 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 37 38 #define NSIM_RING_SIZE 256 39 40 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq) 41 { 42 struct netdevsim *ns = netdev_priv(dev); 43 struct net_device *peer_dev; 44 struct netdevsim *peer_ns; 45 struct netdev_queue *txq; 46 u16 idx; 47 48 idx = rq->napi.index; 49 rcu_read_lock(); 50 peer_ns = rcu_dereference(ns->peer); 51 if (!peer_ns) 52 goto out; 53 54 /* TX device */ 55 peer_dev = peer_ns->netdev; 56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues) 57 goto out; 58 59 txq = netdev_get_tx_queue(peer_dev, idx); 60 if (!netif_tx_queue_stopped(txq)) 61 goto out; 62 63 netif_tx_wake_queue(txq); 64 out: 65 rcu_read_unlock(); 66 } 67 68 static void nsim_stop_tx_queue(struct net_device *tx_dev, 69 struct net_device *rx_dev, 70 struct nsim_rq *rq, 71 u16 idx) 72 { 73 /* If different queues size, do not stop, since it is not 74 * easy to find which TX queue is mapped here 75 */ 76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) 77 return; 78 79 /* rq is the queue on the receive side */ 80 netif_subqueue_try_stop(tx_dev, idx, 81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), 82 NSIM_RING_SIZE / 2); 83 } 84 85 static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, 86 struct nsim_rq *rq, struct sk_buff *skb) 87 { 88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 89 dev_kfree_skb_any(skb); 90 return NET_RX_DROP; 91 } 92 93 skb_queue_tail(&rq->skb_queue, skb); 94 95 /* Stop the peer TX queue avoiding dropping packets later */ 96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) 97 nsim_stop_tx_queue(tx_dev, rx_dev, rq, 98 skb_get_queue_mapping(skb)); 99 100 return NET_RX_SUCCESS; 101 } 102 103 static int nsim_forward_skb(struct net_device *tx_dev, 104 struct net_device *rx_dev, 105 struct sk_buff *skb, 106 struct nsim_rq *rq, 107 struct skb_ext *psp_ext) 108 { 109 int ret; 110 111 ret = __dev_forward_skb(rx_dev, skb); 112 if (ret) { 113 if (psp_ext) 114 __skb_ext_put(psp_ext); 115 return ret; 116 } 117 118 nsim_psp_handle_ext(skb, psp_ext); 119 120 return nsim_napi_rx(tx_dev, rx_dev, rq, skb); 121 } 122 123 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 124 { 125 struct netdevsim *ns = netdev_priv(dev); 126 struct skb_ext *psp_ext = NULL; 127 struct net_device *peer_dev; 128 unsigned int len = skb->len; 129 struct netdevsim *peer_ns; 130 struct netdev_config *cfg; 131 struct nsim_rq *rq; 132 int rxq; 133 int dr; 134 135 rcu_read_lock(); 136 if (!nsim_ipsec_tx(ns, skb)) 137 goto out_drop_any; 138 139 /* Check if loopback mode is enabled */ 140 if (dev->features & NETIF_F_LOOPBACK) { 141 peer_ns = ns; 142 peer_dev = dev; 143 } else { 144 peer_ns = rcu_dereference(ns->peer); 145 if (!peer_ns) 146 goto out_drop_any; 147 peer_dev = peer_ns->netdev; 148 } 149 150 dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext); 151 if (dr) 152 goto out_drop_free; 153 154 rxq = skb_get_queue_mapping(skb); 155 if (rxq >= peer_dev->num_rx_queues) 156 rxq = rxq % peer_dev->num_rx_queues; 157 rq = peer_ns->rq[rxq]; 158 159 cfg = peer_dev->cfg; 160 if (skb_is_nonlinear(skb) && 161 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 162 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 163 cfg->hds_thresh > len))) 164 skb_linearize(skb); 165 166 skb_tx_timestamp(skb); 167 if (unlikely(nsim_forward_skb(dev, peer_dev, 168 skb, rq, psp_ext) == NET_RX_DROP)) 169 goto out_drop_cnt; 170 171 if (!hrtimer_active(&rq->napi_timer)) 172 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 173 174 rcu_read_unlock(); 175 dev_dstats_tx_add(dev, len); 176 return NETDEV_TX_OK; 177 178 out_drop_any: 179 dr = SKB_DROP_REASON_NOT_SPECIFIED; 180 out_drop_free: 181 kfree_skb_reason(skb, dr); 182 out_drop_cnt: 183 rcu_read_unlock(); 184 dev_dstats_tx_dropped(dev); 185 return NETDEV_TX_OK; 186 } 187 188 static void nsim_set_rx_mode(struct net_device *dev) 189 { 190 } 191 192 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 193 { 194 struct netdevsim *ns = netdev_priv(dev); 195 196 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 197 new_mtu > NSIM_XDP_MAX_MTU) 198 return -EBUSY; 199 200 WRITE_ONCE(dev->mtu, new_mtu); 201 202 return 0; 203 } 204 205 static int 206 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 207 { 208 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv); 209 } 210 211 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 212 { 213 struct netdevsim *ns = netdev_priv(dev); 214 struct nsim_dev *nsim_dev = ns->nsim_dev; 215 216 /* Only refuse multicast addresses, zero address can mean unset/any. */ 217 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 218 return -EINVAL; 219 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 220 221 return 0; 222 } 223 224 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 225 u16 vlan, u8 qos, __be16 vlan_proto) 226 { 227 struct netdevsim *ns = netdev_priv(dev); 228 struct nsim_dev *nsim_dev = ns->nsim_dev; 229 230 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 231 return -EINVAL; 232 233 nsim_dev->vfconfigs[vf].vlan = vlan; 234 nsim_dev->vfconfigs[vf].qos = qos; 235 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 236 237 return 0; 238 } 239 240 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 241 { 242 struct netdevsim *ns = netdev_priv(dev); 243 struct nsim_dev *nsim_dev = ns->nsim_dev; 244 245 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 246 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 247 return -EOPNOTSUPP; 248 } 249 250 if (vf >= nsim_dev_get_vfs(nsim_dev)) 251 return -EINVAL; 252 253 nsim_dev->vfconfigs[vf].min_tx_rate = min; 254 nsim_dev->vfconfigs[vf].max_tx_rate = max; 255 256 return 0; 257 } 258 259 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 260 { 261 struct netdevsim *ns = netdev_priv(dev); 262 struct nsim_dev *nsim_dev = ns->nsim_dev; 263 264 if (vf >= nsim_dev_get_vfs(nsim_dev)) 265 return -EINVAL; 266 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 267 268 return 0; 269 } 270 271 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 272 { 273 struct netdevsim *ns = netdev_priv(dev); 274 struct nsim_dev *nsim_dev = ns->nsim_dev; 275 276 if (vf >= nsim_dev_get_vfs(nsim_dev)) 277 return -EINVAL; 278 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 279 280 return 0; 281 } 282 283 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 284 { 285 struct netdevsim *ns = netdev_priv(dev); 286 struct nsim_dev *nsim_dev = ns->nsim_dev; 287 288 if (vf >= nsim_dev_get_vfs(nsim_dev)) 289 return -EINVAL; 290 nsim_dev->vfconfigs[vf].trusted = val; 291 292 return 0; 293 } 294 295 static int 296 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 297 { 298 struct netdevsim *ns = netdev_priv(dev); 299 struct nsim_dev *nsim_dev = ns->nsim_dev; 300 301 if (vf >= nsim_dev_get_vfs(nsim_dev)) 302 return -EINVAL; 303 304 ivi->vf = vf; 305 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 306 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 307 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 308 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 309 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 310 ivi->qos = nsim_dev->vfconfigs[vf].qos; 311 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 312 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 313 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 314 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 315 316 return 0; 317 } 318 319 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 320 { 321 struct netdevsim *ns = netdev_priv(dev); 322 struct nsim_dev *nsim_dev = ns->nsim_dev; 323 324 if (vf >= nsim_dev_get_vfs(nsim_dev)) 325 return -EINVAL; 326 327 switch (state) { 328 case IFLA_VF_LINK_STATE_AUTO: 329 case IFLA_VF_LINK_STATE_ENABLE: 330 case IFLA_VF_LINK_STATE_DISABLE: 331 break; 332 default: 333 return -EINVAL; 334 } 335 336 nsim_dev->vfconfigs[vf].link_state = state; 337 338 return 0; 339 } 340 341 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats) 342 { 343 stats->window_drops = 0; 344 stats->tx_overruns = 0; 345 } 346 347 static int nsim_setup_tc_taprio(struct net_device *dev, 348 struct tc_taprio_qopt_offload *offload) 349 { 350 int err = 0; 351 352 switch (offload->cmd) { 353 case TAPRIO_CMD_REPLACE: 354 case TAPRIO_CMD_DESTROY: 355 break; 356 case TAPRIO_CMD_STATS: 357 nsim_taprio_stats(&offload->stats); 358 break; 359 default: 360 err = -EOPNOTSUPP; 361 } 362 363 return err; 364 } 365 366 static LIST_HEAD(nsim_block_cb_list); 367 368 static int 369 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) 370 { 371 struct netdevsim *ns = netdev_priv(dev); 372 373 switch (type) { 374 case TC_SETUP_QDISC_TAPRIO: 375 return nsim_setup_tc_taprio(dev, type_data); 376 case TC_SETUP_BLOCK: 377 return flow_block_cb_setup_simple(type_data, 378 &nsim_block_cb_list, 379 nsim_setup_tc_block_cb, 380 ns, ns, true); 381 default: 382 return -EOPNOTSUPP; 383 } 384 } 385 386 static int 387 nsim_set_features(struct net_device *dev, netdev_features_t features) 388 { 389 struct netdevsim *ns = netdev_priv(dev); 390 391 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 392 return nsim_bpf_disable_tc(ns); 393 394 return 0; 395 } 396 397 static int nsim_get_iflink(const struct net_device *dev) 398 { 399 struct netdevsim *nsim, *peer; 400 int iflink; 401 402 nsim = netdev_priv(dev); 403 404 rcu_read_lock(); 405 peer = rcu_dereference(nsim->peer); 406 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 407 READ_ONCE(dev->ifindex); 408 rcu_read_unlock(); 409 410 return iflink; 411 } 412 413 static int nsim_rcv(struct nsim_rq *rq, int budget) 414 { 415 struct net_device *dev = rq->napi.dev; 416 struct bpf_prog *xdp_prog; 417 struct netdevsim *ns; 418 struct sk_buff *skb; 419 unsigned int skblen; 420 int i, ret; 421 422 ns = netdev_priv(dev); 423 xdp_prog = READ_ONCE(ns->xdp.prog); 424 425 for (i = 0; i < budget; i++) { 426 if (skb_queue_empty(&rq->skb_queue)) 427 break; 428 429 skb = skb_dequeue(&rq->skb_queue); 430 431 if (xdp_prog) { 432 /* skb might be freed directly by XDP, save the len */ 433 skblen = skb->len; 434 435 if (skb->ip_summed == CHECKSUM_PARTIAL) 436 skb_checksum_help(skb); 437 ret = do_xdp_generic(xdp_prog, &skb); 438 if (ret != XDP_PASS) { 439 dev_dstats_rx_add(dev, skblen); 440 continue; 441 } 442 } 443 444 /* skb might be discard at netif_receive_skb, save the len */ 445 dev_dstats_rx_add(dev, skb->len); 446 napi_gro_receive(&rq->napi, skb); 447 } 448 449 nsim_start_peer_tx_queue(dev, rq); 450 return i; 451 } 452 453 static int nsim_poll(struct napi_struct *napi, int budget) 454 { 455 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 456 int done; 457 458 done = nsim_rcv(rq, budget); 459 if (done < budget) 460 napi_complete_done(napi, done); 461 462 return done; 463 } 464 465 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 466 { 467 struct page_pool_params params = { 468 .order = 0, 469 .pool_size = NSIM_RING_SIZE, 470 .nid = NUMA_NO_NODE, 471 .dev = &napi->dev->dev, 472 .napi = napi, 473 .dma_dir = DMA_BIDIRECTIONAL, 474 .netdev = napi->dev, 475 }; 476 struct page_pool *pool; 477 478 pool = page_pool_create(¶ms); 479 if (IS_ERR(pool)) 480 return PTR_ERR(pool); 481 482 *p = pool; 483 return 0; 484 } 485 486 static int nsim_init_napi(struct netdevsim *ns) 487 { 488 struct net_device *dev = ns->netdev; 489 struct nsim_rq *rq; 490 int err, i; 491 492 for (i = 0; i < dev->num_rx_queues; i++) { 493 rq = ns->rq[i]; 494 495 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 496 } 497 498 for (i = 0; i < dev->num_rx_queues; i++) { 499 rq = ns->rq[i]; 500 501 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 502 if (err) 503 goto err_pp_destroy; 504 } 505 506 return 0; 507 508 err_pp_destroy: 509 while (i--) { 510 page_pool_destroy(ns->rq[i]->page_pool); 511 ns->rq[i]->page_pool = NULL; 512 } 513 514 for (i = 0; i < dev->num_rx_queues; i++) 515 __netif_napi_del_locked(&ns->rq[i]->napi); 516 517 return err; 518 } 519 520 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 521 { 522 struct nsim_rq *rq; 523 524 rq = container_of(timer, struct nsim_rq, napi_timer); 525 napi_schedule(&rq->napi); 526 527 return HRTIMER_NORESTART; 528 } 529 530 static void nsim_rq_timer_init(struct nsim_rq *rq) 531 { 532 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, 533 HRTIMER_MODE_REL); 534 } 535 536 static void nsim_enable_napi(struct netdevsim *ns) 537 { 538 struct net_device *dev = ns->netdev; 539 int i; 540 541 for (i = 0; i < dev->num_rx_queues; i++) { 542 struct nsim_rq *rq = ns->rq[i]; 543 544 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 545 napi_enable_locked(&rq->napi); 546 } 547 } 548 549 static int nsim_open(struct net_device *dev) 550 { 551 struct netdevsim *ns = netdev_priv(dev); 552 struct netdevsim *peer; 553 int err; 554 555 netdev_assert_locked(dev); 556 557 err = nsim_init_napi(ns); 558 if (err) 559 return err; 560 561 nsim_enable_napi(ns); 562 563 peer = rtnl_dereference(ns->peer); 564 if (peer && netif_running(peer->netdev)) { 565 netif_carrier_on(dev); 566 netif_carrier_on(peer->netdev); 567 } 568 569 return 0; 570 } 571 572 static void nsim_del_napi(struct netdevsim *ns) 573 { 574 struct net_device *dev = ns->netdev; 575 int i; 576 577 for (i = 0; i < dev->num_rx_queues; i++) { 578 struct nsim_rq *rq = ns->rq[i]; 579 580 napi_disable_locked(&rq->napi); 581 __netif_napi_del_locked(&rq->napi); 582 } 583 synchronize_net(); 584 585 for (i = 0; i < dev->num_rx_queues; i++) { 586 page_pool_destroy(ns->rq[i]->page_pool); 587 ns->rq[i]->page_pool = NULL; 588 } 589 } 590 591 static int nsim_stop(struct net_device *dev) 592 { 593 struct netdevsim *ns = netdev_priv(dev); 594 struct netdevsim *peer; 595 596 netdev_assert_locked(dev); 597 598 netif_carrier_off(dev); 599 peer = rtnl_dereference(ns->peer); 600 if (peer) 601 netif_carrier_off(peer->netdev); 602 603 nsim_del_napi(ns); 604 605 return 0; 606 } 607 608 static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 609 { 610 struct netdevsim *ns = netdev_priv(dev); 611 612 if (vid >= VLAN_N_VID) 613 return -EINVAL; 614 615 if (proto == htons(ETH_P_8021Q)) 616 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag)); 617 else if (proto == htons(ETH_P_8021AD)) 618 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag)); 619 620 return 0; 621 } 622 623 static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 624 { 625 struct netdevsim *ns = netdev_priv(dev); 626 627 if (vid >= VLAN_N_VID) 628 return -EINVAL; 629 630 if (proto == htons(ETH_P_8021Q)) 631 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag)); 632 else if (proto == htons(ETH_P_8021AD)) 633 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag)); 634 635 return 0; 636 } 637 638 static int nsim_shaper_set(struct net_shaper_binding *binding, 639 const struct net_shaper *shaper, 640 struct netlink_ext_ack *extack) 641 { 642 return 0; 643 } 644 645 static int nsim_shaper_del(struct net_shaper_binding *binding, 646 const struct net_shaper_handle *handle, 647 struct netlink_ext_ack *extack) 648 { 649 return 0; 650 } 651 652 static int nsim_shaper_group(struct net_shaper_binding *binding, 653 int leaves_count, 654 const struct net_shaper *leaves, 655 const struct net_shaper *root, 656 struct netlink_ext_ack *extack) 657 { 658 return 0; 659 } 660 661 static void nsim_shaper_cap(struct net_shaper_binding *binding, 662 enum net_shaper_scope scope, 663 unsigned long *flags) 664 { 665 *flags = ULONG_MAX; 666 } 667 668 static const struct net_shaper_ops nsim_shaper_ops = { 669 .set = nsim_shaper_set, 670 .delete = nsim_shaper_del, 671 .group = nsim_shaper_group, 672 .capabilities = nsim_shaper_cap, 673 }; 674 675 static const struct net_device_ops nsim_netdev_ops = { 676 .ndo_start_xmit = nsim_start_xmit, 677 .ndo_set_rx_mode = nsim_set_rx_mode, 678 .ndo_set_mac_address = eth_mac_addr, 679 .ndo_validate_addr = eth_validate_addr, 680 .ndo_change_mtu = nsim_change_mtu, 681 .ndo_set_vf_mac = nsim_set_vf_mac, 682 .ndo_set_vf_vlan = nsim_set_vf_vlan, 683 .ndo_set_vf_rate = nsim_set_vf_rate, 684 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 685 .ndo_set_vf_trust = nsim_set_vf_trust, 686 .ndo_get_vf_config = nsim_get_vf_config, 687 .ndo_set_vf_link_state = nsim_set_vf_link_state, 688 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 689 .ndo_setup_tc = nsim_setup_tc, 690 .ndo_set_features = nsim_set_features, 691 .ndo_get_iflink = nsim_get_iflink, 692 .ndo_bpf = nsim_bpf, 693 .ndo_open = nsim_open, 694 .ndo_stop = nsim_stop, 695 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 696 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 697 .net_shaper_ops = &nsim_shaper_ops, 698 }; 699 700 static const struct net_device_ops nsim_vf_netdev_ops = { 701 .ndo_start_xmit = nsim_start_xmit, 702 .ndo_set_rx_mode = nsim_set_rx_mode, 703 .ndo_set_mac_address = eth_mac_addr, 704 .ndo_validate_addr = eth_validate_addr, 705 .ndo_change_mtu = nsim_change_mtu, 706 .ndo_setup_tc = nsim_setup_tc, 707 .ndo_set_features = nsim_set_features, 708 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 709 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 710 }; 711 712 /* We don't have true per-queue stats, yet, so do some random fakery here. 713 * Only report stuff for queue 0. 714 */ 715 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 716 struct netdev_queue_stats_rx *stats) 717 { 718 struct rtnl_link_stats64 rtstats = {}; 719 720 if (!idx) 721 dev_get_stats(dev, &rtstats); 722 723 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 724 stats->bytes = rtstats.rx_bytes; 725 } 726 727 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 728 struct netdev_queue_stats_tx *stats) 729 { 730 struct rtnl_link_stats64 rtstats = {}; 731 732 if (!idx) 733 dev_get_stats(dev, &rtstats); 734 735 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 736 stats->bytes = rtstats.tx_bytes; 737 } 738 739 static void nsim_get_base_stats(struct net_device *dev, 740 struct netdev_queue_stats_rx *rx, 741 struct netdev_queue_stats_tx *tx) 742 { 743 struct rtnl_link_stats64 rtstats = {}; 744 745 dev_get_stats(dev, &rtstats); 746 747 rx->packets = !!rtstats.rx_packets; 748 rx->bytes = 0; 749 tx->packets = !!rtstats.tx_packets; 750 tx->bytes = 0; 751 } 752 753 static const struct netdev_stat_ops nsim_stat_ops = { 754 .get_queue_stats_tx = nsim_get_queue_stats_tx, 755 .get_queue_stats_rx = nsim_get_queue_stats_rx, 756 .get_base_stats = nsim_get_base_stats, 757 }; 758 759 static struct nsim_rq *nsim_queue_alloc(void) 760 { 761 struct nsim_rq *rq; 762 763 rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT); 764 if (!rq) 765 return NULL; 766 767 skb_queue_head_init(&rq->skb_queue); 768 nsim_rq_timer_init(rq); 769 return rq; 770 } 771 772 static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq) 773 { 774 hrtimer_cancel(&rq->napi_timer); 775 776 if (rq->skb_queue.qlen) { 777 local_bh_disable(); 778 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); 779 local_bh_enable(); 780 } 781 782 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 783 kfree(rq); 784 } 785 786 /* Queue reset mode is controlled by ns->rq_reset_mode. 787 * - normal - new NAPI new pool (old NAPI enabled when new added) 788 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 789 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 790 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 791 */ 792 struct nsim_queue_mem { 793 struct nsim_rq *rq; 794 struct page_pool *pp; 795 }; 796 797 static int 798 nsim_queue_mem_alloc(struct net_device *dev, 799 struct netdev_queue_config *qcfg, 800 void *per_queue_mem, int idx) 801 { 802 struct nsim_queue_mem *qmem = per_queue_mem; 803 struct netdevsim *ns = netdev_priv(dev); 804 int err; 805 806 if (ns->rq_reset_mode > 3) 807 return -EINVAL; 808 809 if (ns->rq_reset_mode == 1) { 810 if (!netif_running(ns->netdev)) 811 return -ENETDOWN; 812 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 813 } 814 815 qmem->rq = nsim_queue_alloc(); 816 if (!qmem->rq) 817 return -ENOMEM; 818 819 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 820 if (err) 821 goto err_free; 822 823 if (!ns->rq_reset_mode) 824 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 825 idx); 826 827 return 0; 828 829 err_free: 830 nsim_queue_free(dev, qmem->rq); 831 return err; 832 } 833 834 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 835 { 836 struct nsim_queue_mem *qmem = per_queue_mem; 837 struct netdevsim *ns = netdev_priv(dev); 838 839 page_pool_destroy(qmem->pp); 840 if (qmem->rq) { 841 if (!ns->rq_reset_mode) 842 netif_napi_del_locked(&qmem->rq->napi); 843 page_pool_destroy(qmem->rq->page_pool); 844 nsim_queue_free(dev, qmem->rq); 845 } 846 } 847 848 static int 849 nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, 850 void *per_queue_mem, int idx) 851 { 852 struct nsim_queue_mem *qmem = per_queue_mem; 853 struct netdevsim *ns = netdev_priv(dev); 854 855 netdev_assert_locked(dev); 856 857 if (ns->rq_reset_mode == 1) { 858 ns->rq[idx]->page_pool = qmem->pp; 859 napi_enable_locked(&ns->rq[idx]->napi); 860 return 0; 861 } 862 863 /* netif_napi_add()/_del() should normally be called from alloc/free, 864 * here we want to test various call orders. 865 */ 866 if (ns->rq_reset_mode == 2) { 867 netif_napi_del_locked(&ns->rq[idx]->napi); 868 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 869 idx); 870 } else if (ns->rq_reset_mode == 3) { 871 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 872 idx); 873 netif_napi_del_locked(&ns->rq[idx]->napi); 874 } 875 876 ns->rq[idx] = qmem->rq; 877 napi_enable_locked(&ns->rq[idx]->napi); 878 879 return 0; 880 } 881 882 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 883 { 884 struct nsim_queue_mem *qmem = per_queue_mem; 885 struct netdevsim *ns = netdev_priv(dev); 886 887 netdev_assert_locked(dev); 888 889 napi_disable_locked(&ns->rq[idx]->napi); 890 891 if (ns->rq_reset_mode == 1) { 892 qmem->pp = ns->rq[idx]->page_pool; 893 page_pool_disable_direct_recycling(qmem->pp); 894 } else { 895 qmem->rq = ns->rq[idx]; 896 } 897 898 return 0; 899 } 900 901 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 902 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 903 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 904 .ndo_queue_mem_free = nsim_queue_mem_free, 905 .ndo_queue_start = nsim_queue_start, 906 .ndo_queue_stop = nsim_queue_stop, 907 }; 908 909 static ssize_t 910 nsim_qreset_write(struct file *file, const char __user *data, 911 size_t count, loff_t *ppos) 912 { 913 struct netdevsim *ns = file->private_data; 914 unsigned int queue, mode; 915 char buf[32]; 916 ssize_t ret; 917 918 if (count >= sizeof(buf)) 919 return -EINVAL; 920 if (copy_from_user(buf, data, count)) 921 return -EFAULT; 922 buf[count] = '\0'; 923 924 ret = sscanf(buf, "%u %u", &queue, &mode); 925 if (ret != 2) 926 return -EINVAL; 927 928 netdev_lock(ns->netdev); 929 if (queue >= ns->netdev->real_num_rx_queues) { 930 ret = -EINVAL; 931 goto exit_unlock; 932 } 933 934 ns->rq_reset_mode = mode; 935 ret = netdev_rx_queue_restart(ns->netdev, queue); 936 ns->rq_reset_mode = 0; 937 if (ret) 938 goto exit_unlock; 939 940 ret = count; 941 exit_unlock: 942 netdev_unlock(ns->netdev); 943 return ret; 944 } 945 946 static const struct file_operations nsim_qreset_fops = { 947 .open = simple_open, 948 .write = nsim_qreset_write, 949 .owner = THIS_MODULE, 950 }; 951 952 static ssize_t 953 nsim_pp_hold_read(struct file *file, char __user *data, 954 size_t count, loff_t *ppos) 955 { 956 struct netdevsim *ns = file->private_data; 957 char buf[3] = "n\n"; 958 959 if (ns->page) 960 buf[0] = 'y'; 961 962 return simple_read_from_buffer(data, count, ppos, buf, 2); 963 } 964 965 static ssize_t 966 nsim_pp_hold_write(struct file *file, const char __user *data, 967 size_t count, loff_t *ppos) 968 { 969 struct netdevsim *ns = file->private_data; 970 ssize_t ret; 971 bool val; 972 973 ret = kstrtobool_from_user(data, count, &val); 974 if (ret) 975 return ret; 976 977 rtnl_lock(); 978 ret = count; 979 if (val == !!ns->page) 980 goto exit; 981 982 if (!netif_running(ns->netdev) && val) { 983 ret = -ENETDOWN; 984 } else if (val) { 985 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 986 if (!ns->page) 987 ret = -ENOMEM; 988 } else { 989 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 990 ns->page, false); 991 ns->page = NULL; 992 } 993 994 exit: 995 rtnl_unlock(); 996 return ret; 997 } 998 999 static const struct file_operations nsim_pp_hold_fops = { 1000 .open = simple_open, 1001 .read = nsim_pp_hold_read, 1002 .write = nsim_pp_hold_write, 1003 .llseek = generic_file_llseek, 1004 .owner = THIS_MODULE, 1005 }; 1006 1007 static int nsim_vlan_show(struct seq_file *s, void *data) 1008 { 1009 struct netdevsim *ns = s->private; 1010 int vid; 1011 1012 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 1013 seq_printf(s, "ctag %d\n", vid); 1014 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 1015 seq_printf(s, "stag %d\n", vid); 1016 1017 return 0; 1018 } 1019 DEFINE_SHOW_ATTRIBUTE(nsim_vlan); 1020 1021 static void nsim_setup(struct net_device *dev) 1022 { 1023 ether_setup(dev); 1024 eth_hw_addr_random(dev); 1025 1026 dev->flags &= ~IFF_MULTICAST; 1027 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1028 dev->features |= NETIF_F_HIGHDMA | 1029 NETIF_F_SG | 1030 NETIF_F_FRAGLIST | 1031 NETIF_F_HW_CSUM | 1032 NETIF_F_LRO | 1033 NETIF_F_TSO | 1034 NETIF_F_HW_VLAN_CTAG_FILTER | 1035 NETIF_F_HW_VLAN_STAG_FILTER; 1036 dev->hw_features |= NETIF_F_HW_TC | 1037 NETIF_F_SG | 1038 NETIF_F_FRAGLIST | 1039 NETIF_F_HW_CSUM | 1040 NETIF_F_LRO | 1041 NETIF_F_TSO | 1042 NETIF_F_LOOPBACK | 1043 NETIF_F_HW_VLAN_CTAG_FILTER | 1044 NETIF_F_HW_VLAN_STAG_FILTER; 1045 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; 1046 dev->max_mtu = ETH_MAX_MTU; 1047 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD; 1048 } 1049 1050 static int nsim_queue_init(struct netdevsim *ns) 1051 { 1052 struct net_device *dev = ns->netdev; 1053 int i; 1054 1055 ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT); 1056 if (!ns->rq) 1057 return -ENOMEM; 1058 1059 for (i = 0; i < dev->num_rx_queues; i++) { 1060 ns->rq[i] = nsim_queue_alloc(); 1061 if (!ns->rq[i]) 1062 goto err_free_prev; 1063 } 1064 1065 return 0; 1066 1067 err_free_prev: 1068 while (i--) 1069 kfree(ns->rq[i]); 1070 kfree(ns->rq); 1071 return -ENOMEM; 1072 } 1073 1074 static void nsim_queue_uninit(struct netdevsim *ns) 1075 { 1076 struct net_device *dev = ns->netdev; 1077 int i; 1078 1079 for (i = 0; i < dev->num_rx_queues; i++) 1080 nsim_queue_free(dev, ns->rq[i]); 1081 1082 kfree(ns->rq); 1083 ns->rq = NULL; 1084 } 1085 1086 static int nsim_init_netdevsim(struct netdevsim *ns) 1087 { 1088 struct netdevsim *peer; 1089 struct mock_phc *phc; 1090 int err; 1091 1092 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 1093 if (IS_ERR(phc)) 1094 return PTR_ERR(phc); 1095 1096 ns->phc = phc; 1097 ns->netdev->netdev_ops = &nsim_netdev_ops; 1098 ns->netdev->stat_ops = &nsim_stat_ops; 1099 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 1100 netdev_lockdep_set_classes(ns->netdev); 1101 1102 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 1103 if (err) 1104 goto err_phc_destroy; 1105 1106 rtnl_lock(); 1107 err = nsim_queue_init(ns); 1108 if (err) 1109 goto err_utn_destroy; 1110 1111 err = nsim_bpf_init(ns); 1112 if (err) 1113 goto err_rq_destroy; 1114 1115 nsim_macsec_init(ns); 1116 nsim_ipsec_init(ns); 1117 1118 err = register_netdevice(ns->netdev); 1119 if (err) 1120 goto err_ipsec_teardown; 1121 rtnl_unlock(); 1122 1123 err = nsim_psp_init(ns); 1124 if (err) 1125 goto err_unregister_netdev; 1126 1127 if (IS_ENABLED(CONFIG_DEBUG_NET)) { 1128 ns->nb.notifier_call = netdev_debug_event; 1129 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1130 &ns->nn)) 1131 ns->nb.notifier_call = NULL; 1132 } 1133 1134 return 0; 1135 1136 err_unregister_netdev: 1137 rtnl_lock(); 1138 peer = rtnl_dereference(ns->peer); 1139 if (peer) 1140 RCU_INIT_POINTER(peer->peer, NULL); 1141 RCU_INIT_POINTER(ns->peer, NULL); 1142 unregister_netdevice(ns->netdev); 1143 err_ipsec_teardown: 1144 nsim_ipsec_teardown(ns); 1145 nsim_macsec_teardown(ns); 1146 nsim_bpf_uninit(ns); 1147 err_rq_destroy: 1148 nsim_queue_uninit(ns); 1149 err_utn_destroy: 1150 rtnl_unlock(); 1151 nsim_udp_tunnels_info_destroy(ns->netdev); 1152 err_phc_destroy: 1153 mock_phc_destroy(ns->phc); 1154 return err; 1155 } 1156 1157 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 1158 { 1159 int err; 1160 1161 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 1162 rtnl_lock(); 1163 err = register_netdevice(ns->netdev); 1164 rtnl_unlock(); 1165 return err; 1166 } 1167 1168 static void nsim_exit_netdevsim(struct netdevsim *ns) 1169 { 1170 nsim_udp_tunnels_info_destroy(ns->netdev); 1171 mock_phc_destroy(ns->phc); 1172 } 1173 1174 struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, 1175 struct nsim_dev_port *nsim_dev_port, 1176 u8 perm_addr[ETH_ALEN]) 1177 { 1178 struct net_device *dev; 1179 struct netdevsim *ns; 1180 int err; 1181 1182 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1183 nsim_dev->nsim_bus_dev->num_queues); 1184 if (!dev) 1185 return ERR_PTR(-ENOMEM); 1186 1187 if (perm_addr) 1188 memcpy(dev->perm_addr, perm_addr, ETH_ALEN); 1189 1190 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1191 ns = netdev_priv(dev); 1192 ns->netdev = dev; 1193 ns->nsim_dev = nsim_dev; 1194 ns->nsim_dev_port = nsim_dev_port; 1195 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1196 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1197 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1198 nsim_ethtool_init(ns); 1199 if (nsim_dev_port_is_pf(nsim_dev_port)) 1200 err = nsim_init_netdevsim(ns); 1201 else 1202 err = nsim_init_netdevsim_vf(ns); 1203 if (err) 1204 goto err_free_netdev; 1205 1206 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1207 ns, &nsim_pp_hold_fops); 1208 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1209 nsim_dev_port->ddir, ns, 1210 &nsim_qreset_fops); 1211 ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir, 1212 ns, &nsim_vlan_fops); 1213 return ns; 1214 1215 err_free_netdev: 1216 free_netdev(dev); 1217 return ERR_PTR(err); 1218 } 1219 1220 void nsim_destroy(struct netdevsim *ns) 1221 { 1222 struct net_device *dev = ns->netdev; 1223 struct netdevsim *peer; 1224 u16 vid; 1225 1226 debugfs_remove(ns->vlan_dfs); 1227 debugfs_remove(ns->qr_dfs); 1228 debugfs_remove(ns->pp_dfs); 1229 1230 if (ns->nb.notifier_call) 1231 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1232 &ns->nn); 1233 1234 nsim_psp_uninit(ns); 1235 1236 rtnl_lock(); 1237 peer = rtnl_dereference(ns->peer); 1238 if (peer) 1239 RCU_INIT_POINTER(peer->peer, NULL); 1240 RCU_INIT_POINTER(ns->peer, NULL); 1241 unregister_netdevice(dev); 1242 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1243 nsim_macsec_teardown(ns); 1244 nsim_ipsec_teardown(ns); 1245 nsim_bpf_uninit(ns); 1246 nsim_queue_uninit(ns); 1247 } 1248 rtnl_unlock(); 1249 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1250 nsim_exit_netdevsim(ns); 1251 1252 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 1253 WARN_ON_ONCE(1); 1254 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 1255 WARN_ON_ONCE(1); 1256 1257 /* Put this intentionally late to exercise the orphaning path */ 1258 if (ns->page) { 1259 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 1260 ns->page, false); 1261 ns->page = NULL; 1262 } 1263 1264 free_netdev(dev); 1265 } 1266 1267 bool netdev_is_nsim(struct net_device *dev) 1268 { 1269 return dev->netdev_ops == &nsim_netdev_ops; 1270 } 1271 1272 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1273 struct netlink_ext_ack *extack) 1274 { 1275 NL_SET_ERR_MSG_MOD(extack, 1276 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1277 return -EOPNOTSUPP; 1278 } 1279 1280 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1281 .kind = DRV_NAME, 1282 .validate = nsim_validate, 1283 }; 1284 1285 static int __init nsim_module_init(void) 1286 { 1287 int err; 1288 1289 err = nsim_dev_init(); 1290 if (err) 1291 return err; 1292 1293 err = nsim_bus_init(); 1294 if (err) 1295 goto err_dev_exit; 1296 1297 err = rtnl_link_register(&nsim_link_ops); 1298 if (err) 1299 goto err_bus_exit; 1300 1301 return 0; 1302 1303 err_bus_exit: 1304 nsim_bus_exit(); 1305 err_dev_exit: 1306 nsim_dev_exit(); 1307 return err; 1308 } 1309 1310 static void __exit nsim_module_exit(void) 1311 { 1312 rtnl_link_unregister(&nsim_link_ops); 1313 nsim_bus_exit(); 1314 nsim_dev_exit(); 1315 } 1316 1317 module_init(nsim_module_init); 1318 module_exit(nsim_module_exit); 1319 MODULE_LICENSE("GPL"); 1320 MODULE_DESCRIPTION("Simulated networking device for testing"); 1321 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1322