1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 #include <net/busy_poll.h> 33 34 #include "netdevsim.h" 35 36 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 37 38 #define NSIM_RING_SIZE 256 39 40 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq) 41 { 42 struct netdevsim *ns = netdev_priv(dev); 43 struct net_device *peer_dev; 44 struct netdevsim *peer_ns; 45 struct netdev_queue *txq; 46 u16 idx; 47 48 idx = rq->napi.index; 49 rcu_read_lock(); 50 peer_ns = rcu_dereference(ns->peer); 51 if (!peer_ns) 52 goto out; 53 54 /* TX device */ 55 peer_dev = peer_ns->netdev; 56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues) 57 goto out; 58 59 txq = netdev_get_tx_queue(peer_dev, idx); 60 if (!netif_tx_queue_stopped(txq)) 61 goto out; 62 63 netif_tx_wake_queue(txq); 64 out: 65 rcu_read_unlock(); 66 } 67 68 static void nsim_stop_tx_queue(struct net_device *tx_dev, 69 struct net_device *rx_dev, 70 struct nsim_rq *rq, 71 u16 idx) 72 { 73 /* If different queues size, do not stop, since it is not 74 * easy to find which TX queue is mapped here 75 */ 76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) 77 return; 78 79 /* rq is the queue on the receive side */ 80 netif_subqueue_try_stop(tx_dev, idx, 81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), 82 NSIM_RING_SIZE / 2); 83 } 84 85 static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, 86 struct nsim_rq *rq, struct sk_buff *skb) 87 { 88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 89 dev_kfree_skb_any(skb); 90 return NET_RX_DROP; 91 } 92 93 skb_queue_tail(&rq->skb_queue, skb); 94 95 /* Stop the peer TX queue avoiding dropping packets later */ 96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) 97 nsim_stop_tx_queue(tx_dev, rx_dev, rq, 98 skb_get_queue_mapping(skb)); 99 100 return NET_RX_SUCCESS; 101 } 102 103 static int nsim_forward_skb(struct net_device *tx_dev, 104 struct net_device *rx_dev, 105 struct sk_buff *skb, 106 struct nsim_rq *rq) 107 { 108 return __dev_forward_skb(rx_dev, skb) ?: 109 nsim_napi_rx(tx_dev, rx_dev, rq, skb); 110 } 111 112 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 113 { 114 struct netdevsim *ns = netdev_priv(dev); 115 struct net_device *peer_dev; 116 unsigned int len = skb->len; 117 struct netdevsim *peer_ns; 118 struct netdev_config *cfg; 119 struct nsim_rq *rq; 120 int rxq; 121 122 rcu_read_lock(); 123 if (!nsim_ipsec_tx(ns, skb)) 124 goto out_drop_free; 125 126 peer_ns = rcu_dereference(ns->peer); 127 if (!peer_ns) 128 goto out_drop_free; 129 130 peer_dev = peer_ns->netdev; 131 rxq = skb_get_queue_mapping(skb); 132 if (rxq >= peer_dev->num_rx_queues) 133 rxq = rxq % peer_dev->num_rx_queues; 134 rq = peer_ns->rq[rxq]; 135 136 cfg = peer_dev->cfg; 137 if (skb_is_nonlinear(skb) && 138 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 139 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 140 cfg->hds_thresh > len))) 141 skb_linearize(skb); 142 143 skb_tx_timestamp(skb); 144 if (unlikely(nsim_forward_skb(dev, peer_dev, skb, rq) == NET_RX_DROP)) 145 goto out_drop_cnt; 146 147 if (!hrtimer_active(&rq->napi_timer)) 148 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 149 150 rcu_read_unlock(); 151 dev_dstats_tx_add(dev, len); 152 return NETDEV_TX_OK; 153 154 out_drop_free: 155 dev_kfree_skb(skb); 156 out_drop_cnt: 157 rcu_read_unlock(); 158 dev_dstats_tx_dropped(dev); 159 return NETDEV_TX_OK; 160 } 161 162 static void nsim_set_rx_mode(struct net_device *dev) 163 { 164 } 165 166 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 167 { 168 struct netdevsim *ns = netdev_priv(dev); 169 170 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 171 new_mtu > NSIM_XDP_MAX_MTU) 172 return -EBUSY; 173 174 WRITE_ONCE(dev->mtu, new_mtu); 175 176 return 0; 177 } 178 179 static int 180 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 181 { 182 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv); 183 } 184 185 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 186 { 187 struct netdevsim *ns = netdev_priv(dev); 188 struct nsim_dev *nsim_dev = ns->nsim_dev; 189 190 /* Only refuse multicast addresses, zero address can mean unset/any. */ 191 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 192 return -EINVAL; 193 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 194 195 return 0; 196 } 197 198 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 199 u16 vlan, u8 qos, __be16 vlan_proto) 200 { 201 struct netdevsim *ns = netdev_priv(dev); 202 struct nsim_dev *nsim_dev = ns->nsim_dev; 203 204 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 205 return -EINVAL; 206 207 nsim_dev->vfconfigs[vf].vlan = vlan; 208 nsim_dev->vfconfigs[vf].qos = qos; 209 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 210 211 return 0; 212 } 213 214 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 215 { 216 struct netdevsim *ns = netdev_priv(dev); 217 struct nsim_dev *nsim_dev = ns->nsim_dev; 218 219 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 220 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 221 return -EOPNOTSUPP; 222 } 223 224 if (vf >= nsim_dev_get_vfs(nsim_dev)) 225 return -EINVAL; 226 227 nsim_dev->vfconfigs[vf].min_tx_rate = min; 228 nsim_dev->vfconfigs[vf].max_tx_rate = max; 229 230 return 0; 231 } 232 233 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 234 { 235 struct netdevsim *ns = netdev_priv(dev); 236 struct nsim_dev *nsim_dev = ns->nsim_dev; 237 238 if (vf >= nsim_dev_get_vfs(nsim_dev)) 239 return -EINVAL; 240 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 241 242 return 0; 243 } 244 245 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 246 { 247 struct netdevsim *ns = netdev_priv(dev); 248 struct nsim_dev *nsim_dev = ns->nsim_dev; 249 250 if (vf >= nsim_dev_get_vfs(nsim_dev)) 251 return -EINVAL; 252 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 253 254 return 0; 255 } 256 257 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 258 { 259 struct netdevsim *ns = netdev_priv(dev); 260 struct nsim_dev *nsim_dev = ns->nsim_dev; 261 262 if (vf >= nsim_dev_get_vfs(nsim_dev)) 263 return -EINVAL; 264 nsim_dev->vfconfigs[vf].trusted = val; 265 266 return 0; 267 } 268 269 static int 270 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 271 { 272 struct netdevsim *ns = netdev_priv(dev); 273 struct nsim_dev *nsim_dev = ns->nsim_dev; 274 275 if (vf >= nsim_dev_get_vfs(nsim_dev)) 276 return -EINVAL; 277 278 ivi->vf = vf; 279 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 280 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 281 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 282 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 283 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 284 ivi->qos = nsim_dev->vfconfigs[vf].qos; 285 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 286 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 287 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 288 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 289 290 return 0; 291 } 292 293 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 294 { 295 struct netdevsim *ns = netdev_priv(dev); 296 struct nsim_dev *nsim_dev = ns->nsim_dev; 297 298 if (vf >= nsim_dev_get_vfs(nsim_dev)) 299 return -EINVAL; 300 301 switch (state) { 302 case IFLA_VF_LINK_STATE_AUTO: 303 case IFLA_VF_LINK_STATE_ENABLE: 304 case IFLA_VF_LINK_STATE_DISABLE: 305 break; 306 default: 307 return -EINVAL; 308 } 309 310 nsim_dev->vfconfigs[vf].link_state = state; 311 312 return 0; 313 } 314 315 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats) 316 { 317 stats->window_drops = 0; 318 stats->tx_overruns = 0; 319 } 320 321 static int nsim_setup_tc_taprio(struct net_device *dev, 322 struct tc_taprio_qopt_offload *offload) 323 { 324 int err = 0; 325 326 switch (offload->cmd) { 327 case TAPRIO_CMD_REPLACE: 328 case TAPRIO_CMD_DESTROY: 329 break; 330 case TAPRIO_CMD_STATS: 331 nsim_taprio_stats(&offload->stats); 332 break; 333 default: 334 err = -EOPNOTSUPP; 335 } 336 337 return err; 338 } 339 340 static LIST_HEAD(nsim_block_cb_list); 341 342 static int 343 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) 344 { 345 struct netdevsim *ns = netdev_priv(dev); 346 347 switch (type) { 348 case TC_SETUP_QDISC_TAPRIO: 349 return nsim_setup_tc_taprio(dev, type_data); 350 case TC_SETUP_BLOCK: 351 return flow_block_cb_setup_simple(type_data, 352 &nsim_block_cb_list, 353 nsim_setup_tc_block_cb, 354 ns, ns, true); 355 default: 356 return -EOPNOTSUPP; 357 } 358 } 359 360 static int 361 nsim_set_features(struct net_device *dev, netdev_features_t features) 362 { 363 struct netdevsim *ns = netdev_priv(dev); 364 365 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 366 return nsim_bpf_disable_tc(ns); 367 368 return 0; 369 } 370 371 static int nsim_get_iflink(const struct net_device *dev) 372 { 373 struct netdevsim *nsim, *peer; 374 int iflink; 375 376 nsim = netdev_priv(dev); 377 378 rcu_read_lock(); 379 peer = rcu_dereference(nsim->peer); 380 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 381 READ_ONCE(dev->ifindex); 382 rcu_read_unlock(); 383 384 return iflink; 385 } 386 387 static int nsim_rcv(struct nsim_rq *rq, int budget) 388 { 389 struct net_device *dev = rq->napi.dev; 390 struct bpf_prog *xdp_prog; 391 struct netdevsim *ns; 392 struct sk_buff *skb; 393 unsigned int skblen; 394 int i, ret; 395 396 ns = netdev_priv(dev); 397 xdp_prog = READ_ONCE(ns->xdp.prog); 398 399 for (i = 0; i < budget; i++) { 400 if (skb_queue_empty(&rq->skb_queue)) 401 break; 402 403 skb = skb_dequeue(&rq->skb_queue); 404 405 if (xdp_prog) { 406 /* skb might be freed directly by XDP, save the len */ 407 skblen = skb->len; 408 409 if (skb->ip_summed == CHECKSUM_PARTIAL) 410 skb_checksum_help(skb); 411 ret = do_xdp_generic(xdp_prog, &skb); 412 if (ret != XDP_PASS) { 413 dev_dstats_rx_add(dev, skblen); 414 continue; 415 } 416 } 417 418 /* skb might be discard at netif_receive_skb, save the len */ 419 skblen = skb->len; 420 skb_mark_napi_id(skb, &rq->napi); 421 ret = netif_receive_skb(skb); 422 if (ret == NET_RX_SUCCESS) 423 dev_dstats_rx_add(dev, skblen); 424 else 425 dev_dstats_rx_dropped(dev); 426 } 427 428 nsim_start_peer_tx_queue(dev, rq); 429 return i; 430 } 431 432 static int nsim_poll(struct napi_struct *napi, int budget) 433 { 434 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 435 int done; 436 437 done = nsim_rcv(rq, budget); 438 if (done < budget) 439 napi_complete_done(napi, done); 440 441 return done; 442 } 443 444 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 445 { 446 struct page_pool_params params = { 447 .order = 0, 448 .pool_size = NSIM_RING_SIZE, 449 .nid = NUMA_NO_NODE, 450 .dev = &napi->dev->dev, 451 .napi = napi, 452 .dma_dir = DMA_BIDIRECTIONAL, 453 .netdev = napi->dev, 454 }; 455 struct page_pool *pool; 456 457 pool = page_pool_create(¶ms); 458 if (IS_ERR(pool)) 459 return PTR_ERR(pool); 460 461 *p = pool; 462 return 0; 463 } 464 465 static int nsim_init_napi(struct netdevsim *ns) 466 { 467 struct net_device *dev = ns->netdev; 468 struct nsim_rq *rq; 469 int err, i; 470 471 for (i = 0; i < dev->num_rx_queues; i++) { 472 rq = ns->rq[i]; 473 474 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 475 } 476 477 for (i = 0; i < dev->num_rx_queues; i++) { 478 rq = ns->rq[i]; 479 480 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 481 if (err) 482 goto err_pp_destroy; 483 } 484 485 return 0; 486 487 err_pp_destroy: 488 while (i--) { 489 page_pool_destroy(ns->rq[i]->page_pool); 490 ns->rq[i]->page_pool = NULL; 491 } 492 493 for (i = 0; i < dev->num_rx_queues; i++) 494 __netif_napi_del_locked(&ns->rq[i]->napi); 495 496 return err; 497 } 498 499 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 500 { 501 struct nsim_rq *rq; 502 503 rq = container_of(timer, struct nsim_rq, napi_timer); 504 napi_schedule(&rq->napi); 505 506 return HRTIMER_NORESTART; 507 } 508 509 static void nsim_rq_timer_init(struct nsim_rq *rq) 510 { 511 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, 512 HRTIMER_MODE_REL); 513 } 514 515 static void nsim_enable_napi(struct netdevsim *ns) 516 { 517 struct net_device *dev = ns->netdev; 518 int i; 519 520 for (i = 0; i < dev->num_rx_queues; i++) { 521 struct nsim_rq *rq = ns->rq[i]; 522 523 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 524 napi_enable_locked(&rq->napi); 525 } 526 } 527 528 static int nsim_open(struct net_device *dev) 529 { 530 struct netdevsim *ns = netdev_priv(dev); 531 int err; 532 533 netdev_assert_locked(dev); 534 535 err = nsim_init_napi(ns); 536 if (err) 537 return err; 538 539 nsim_enable_napi(ns); 540 541 return 0; 542 } 543 544 static void nsim_del_napi(struct netdevsim *ns) 545 { 546 struct net_device *dev = ns->netdev; 547 int i; 548 549 for (i = 0; i < dev->num_rx_queues; i++) { 550 struct nsim_rq *rq = ns->rq[i]; 551 552 napi_disable_locked(&rq->napi); 553 __netif_napi_del_locked(&rq->napi); 554 } 555 synchronize_net(); 556 557 for (i = 0; i < dev->num_rx_queues; i++) { 558 page_pool_destroy(ns->rq[i]->page_pool); 559 ns->rq[i]->page_pool = NULL; 560 } 561 } 562 563 static int nsim_stop(struct net_device *dev) 564 { 565 struct netdevsim *ns = netdev_priv(dev); 566 struct netdevsim *peer; 567 568 netdev_assert_locked(dev); 569 570 netif_carrier_off(dev); 571 peer = rtnl_dereference(ns->peer); 572 if (peer) 573 netif_carrier_off(peer->netdev); 574 575 nsim_del_napi(ns); 576 577 return 0; 578 } 579 580 static int nsim_shaper_set(struct net_shaper_binding *binding, 581 const struct net_shaper *shaper, 582 struct netlink_ext_ack *extack) 583 { 584 return 0; 585 } 586 587 static int nsim_shaper_del(struct net_shaper_binding *binding, 588 const struct net_shaper_handle *handle, 589 struct netlink_ext_ack *extack) 590 { 591 return 0; 592 } 593 594 static int nsim_shaper_group(struct net_shaper_binding *binding, 595 int leaves_count, 596 const struct net_shaper *leaves, 597 const struct net_shaper *root, 598 struct netlink_ext_ack *extack) 599 { 600 return 0; 601 } 602 603 static void nsim_shaper_cap(struct net_shaper_binding *binding, 604 enum net_shaper_scope scope, 605 unsigned long *flags) 606 { 607 *flags = ULONG_MAX; 608 } 609 610 static const struct net_shaper_ops nsim_shaper_ops = { 611 .set = nsim_shaper_set, 612 .delete = nsim_shaper_del, 613 .group = nsim_shaper_group, 614 .capabilities = nsim_shaper_cap, 615 }; 616 617 static const struct net_device_ops nsim_netdev_ops = { 618 .ndo_start_xmit = nsim_start_xmit, 619 .ndo_set_rx_mode = nsim_set_rx_mode, 620 .ndo_set_mac_address = eth_mac_addr, 621 .ndo_validate_addr = eth_validate_addr, 622 .ndo_change_mtu = nsim_change_mtu, 623 .ndo_set_vf_mac = nsim_set_vf_mac, 624 .ndo_set_vf_vlan = nsim_set_vf_vlan, 625 .ndo_set_vf_rate = nsim_set_vf_rate, 626 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 627 .ndo_set_vf_trust = nsim_set_vf_trust, 628 .ndo_get_vf_config = nsim_get_vf_config, 629 .ndo_set_vf_link_state = nsim_set_vf_link_state, 630 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 631 .ndo_setup_tc = nsim_setup_tc, 632 .ndo_set_features = nsim_set_features, 633 .ndo_get_iflink = nsim_get_iflink, 634 .ndo_bpf = nsim_bpf, 635 .ndo_open = nsim_open, 636 .ndo_stop = nsim_stop, 637 .net_shaper_ops = &nsim_shaper_ops, 638 }; 639 640 static const struct net_device_ops nsim_vf_netdev_ops = { 641 .ndo_start_xmit = nsim_start_xmit, 642 .ndo_set_rx_mode = nsim_set_rx_mode, 643 .ndo_set_mac_address = eth_mac_addr, 644 .ndo_validate_addr = eth_validate_addr, 645 .ndo_change_mtu = nsim_change_mtu, 646 .ndo_setup_tc = nsim_setup_tc, 647 .ndo_set_features = nsim_set_features, 648 }; 649 650 /* We don't have true per-queue stats, yet, so do some random fakery here. 651 * Only report stuff for queue 0. 652 */ 653 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 654 struct netdev_queue_stats_rx *stats) 655 { 656 struct rtnl_link_stats64 rtstats = {}; 657 658 if (!idx) 659 dev_get_stats(dev, &rtstats); 660 661 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 662 stats->bytes = rtstats.rx_bytes; 663 } 664 665 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 666 struct netdev_queue_stats_tx *stats) 667 { 668 struct rtnl_link_stats64 rtstats = {}; 669 670 if (!idx) 671 dev_get_stats(dev, &rtstats); 672 673 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 674 stats->bytes = rtstats.tx_bytes; 675 } 676 677 static void nsim_get_base_stats(struct net_device *dev, 678 struct netdev_queue_stats_rx *rx, 679 struct netdev_queue_stats_tx *tx) 680 { 681 struct rtnl_link_stats64 rtstats = {}; 682 683 dev_get_stats(dev, &rtstats); 684 685 rx->packets = !!rtstats.rx_packets; 686 rx->bytes = 0; 687 tx->packets = !!rtstats.tx_packets; 688 tx->bytes = 0; 689 } 690 691 static const struct netdev_stat_ops nsim_stat_ops = { 692 .get_queue_stats_tx = nsim_get_queue_stats_tx, 693 .get_queue_stats_rx = nsim_get_queue_stats_rx, 694 .get_base_stats = nsim_get_base_stats, 695 }; 696 697 static struct nsim_rq *nsim_queue_alloc(void) 698 { 699 struct nsim_rq *rq; 700 701 rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT); 702 if (!rq) 703 return NULL; 704 705 skb_queue_head_init(&rq->skb_queue); 706 nsim_rq_timer_init(rq); 707 return rq; 708 } 709 710 static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq) 711 { 712 hrtimer_cancel(&rq->napi_timer); 713 local_bh_disable(); 714 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); 715 local_bh_enable(); 716 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 717 kfree(rq); 718 } 719 720 /* Queue reset mode is controlled by ns->rq_reset_mode. 721 * - normal - new NAPI new pool (old NAPI enabled when new added) 722 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 723 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 724 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 725 */ 726 struct nsim_queue_mem { 727 struct nsim_rq *rq; 728 struct page_pool *pp; 729 }; 730 731 static int 732 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx) 733 { 734 struct nsim_queue_mem *qmem = per_queue_mem; 735 struct netdevsim *ns = netdev_priv(dev); 736 int err; 737 738 if (ns->rq_reset_mode > 3) 739 return -EINVAL; 740 741 if (ns->rq_reset_mode == 1) { 742 if (!netif_running(ns->netdev)) 743 return -ENETDOWN; 744 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 745 } 746 747 qmem->rq = nsim_queue_alloc(); 748 if (!qmem->rq) 749 return -ENOMEM; 750 751 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 752 if (err) 753 goto err_free; 754 755 if (!ns->rq_reset_mode) 756 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 757 idx); 758 759 return 0; 760 761 err_free: 762 nsim_queue_free(dev, qmem->rq); 763 return err; 764 } 765 766 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 767 { 768 struct nsim_queue_mem *qmem = per_queue_mem; 769 struct netdevsim *ns = netdev_priv(dev); 770 771 page_pool_destroy(qmem->pp); 772 if (qmem->rq) { 773 if (!ns->rq_reset_mode) 774 netif_napi_del_locked(&qmem->rq->napi); 775 page_pool_destroy(qmem->rq->page_pool); 776 nsim_queue_free(dev, qmem->rq); 777 } 778 } 779 780 static int 781 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) 782 { 783 struct nsim_queue_mem *qmem = per_queue_mem; 784 struct netdevsim *ns = netdev_priv(dev); 785 786 netdev_assert_locked(dev); 787 788 if (ns->rq_reset_mode == 1) { 789 ns->rq[idx]->page_pool = qmem->pp; 790 napi_enable_locked(&ns->rq[idx]->napi); 791 return 0; 792 } 793 794 /* netif_napi_add()/_del() should normally be called from alloc/free, 795 * here we want to test various call orders. 796 */ 797 if (ns->rq_reset_mode == 2) { 798 netif_napi_del_locked(&ns->rq[idx]->napi); 799 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 800 idx); 801 } else if (ns->rq_reset_mode == 3) { 802 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 803 idx); 804 netif_napi_del_locked(&ns->rq[idx]->napi); 805 } 806 807 ns->rq[idx] = qmem->rq; 808 napi_enable_locked(&ns->rq[idx]->napi); 809 810 return 0; 811 } 812 813 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 814 { 815 struct nsim_queue_mem *qmem = per_queue_mem; 816 struct netdevsim *ns = netdev_priv(dev); 817 818 netdev_assert_locked(dev); 819 820 napi_disable_locked(&ns->rq[idx]->napi); 821 822 if (ns->rq_reset_mode == 1) { 823 qmem->pp = ns->rq[idx]->page_pool; 824 page_pool_disable_direct_recycling(qmem->pp); 825 } else { 826 qmem->rq = ns->rq[idx]; 827 } 828 829 return 0; 830 } 831 832 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 833 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 834 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 835 .ndo_queue_mem_free = nsim_queue_mem_free, 836 .ndo_queue_start = nsim_queue_start, 837 .ndo_queue_stop = nsim_queue_stop, 838 }; 839 840 static ssize_t 841 nsim_qreset_write(struct file *file, const char __user *data, 842 size_t count, loff_t *ppos) 843 { 844 struct netdevsim *ns = file->private_data; 845 unsigned int queue, mode; 846 char buf[32]; 847 ssize_t ret; 848 849 if (count >= sizeof(buf)) 850 return -EINVAL; 851 if (copy_from_user(buf, data, count)) 852 return -EFAULT; 853 buf[count] = '\0'; 854 855 ret = sscanf(buf, "%u %u", &queue, &mode); 856 if (ret != 2) 857 return -EINVAL; 858 859 netdev_lock(ns->netdev); 860 if (queue >= ns->netdev->real_num_rx_queues) { 861 ret = -EINVAL; 862 goto exit_unlock; 863 } 864 865 ns->rq_reset_mode = mode; 866 ret = netdev_rx_queue_restart(ns->netdev, queue); 867 ns->rq_reset_mode = 0; 868 if (ret) 869 goto exit_unlock; 870 871 ret = count; 872 exit_unlock: 873 netdev_unlock(ns->netdev); 874 return ret; 875 } 876 877 static const struct file_operations nsim_qreset_fops = { 878 .open = simple_open, 879 .write = nsim_qreset_write, 880 .owner = THIS_MODULE, 881 }; 882 883 static ssize_t 884 nsim_pp_hold_read(struct file *file, char __user *data, 885 size_t count, loff_t *ppos) 886 { 887 struct netdevsim *ns = file->private_data; 888 char buf[3] = "n\n"; 889 890 if (ns->page) 891 buf[0] = 'y'; 892 893 return simple_read_from_buffer(data, count, ppos, buf, 2); 894 } 895 896 static ssize_t 897 nsim_pp_hold_write(struct file *file, const char __user *data, 898 size_t count, loff_t *ppos) 899 { 900 struct netdevsim *ns = file->private_data; 901 ssize_t ret; 902 bool val; 903 904 ret = kstrtobool_from_user(data, count, &val); 905 if (ret) 906 return ret; 907 908 rtnl_lock(); 909 ret = count; 910 if (val == !!ns->page) 911 goto exit; 912 913 if (!netif_running(ns->netdev) && val) { 914 ret = -ENETDOWN; 915 } else if (val) { 916 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 917 if (!ns->page) 918 ret = -ENOMEM; 919 } else { 920 page_pool_put_full_page(ns->page->pp, ns->page, false); 921 ns->page = NULL; 922 } 923 924 exit: 925 rtnl_unlock(); 926 return ret; 927 } 928 929 static const struct file_operations nsim_pp_hold_fops = { 930 .open = simple_open, 931 .read = nsim_pp_hold_read, 932 .write = nsim_pp_hold_write, 933 .llseek = generic_file_llseek, 934 .owner = THIS_MODULE, 935 }; 936 937 static void nsim_setup(struct net_device *dev) 938 { 939 ether_setup(dev); 940 eth_hw_addr_random(dev); 941 942 dev->flags &= ~IFF_MULTICAST; 943 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 944 dev->features |= NETIF_F_HIGHDMA | 945 NETIF_F_SG | 946 NETIF_F_FRAGLIST | 947 NETIF_F_HW_CSUM | 948 NETIF_F_LRO | 949 NETIF_F_TSO; 950 dev->hw_features |= NETIF_F_HW_TC | 951 NETIF_F_SG | 952 NETIF_F_FRAGLIST | 953 NETIF_F_HW_CSUM | 954 NETIF_F_LRO | 955 NETIF_F_TSO; 956 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; 957 dev->max_mtu = ETH_MAX_MTU; 958 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD; 959 } 960 961 static int nsim_queue_init(struct netdevsim *ns) 962 { 963 struct net_device *dev = ns->netdev; 964 int i; 965 966 ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq), 967 GFP_KERNEL_ACCOUNT); 968 if (!ns->rq) 969 return -ENOMEM; 970 971 for (i = 0; i < dev->num_rx_queues; i++) { 972 ns->rq[i] = nsim_queue_alloc(); 973 if (!ns->rq[i]) 974 goto err_free_prev; 975 } 976 977 return 0; 978 979 err_free_prev: 980 while (i--) 981 kfree(ns->rq[i]); 982 kfree(ns->rq); 983 return -ENOMEM; 984 } 985 986 static void nsim_queue_uninit(struct netdevsim *ns) 987 { 988 struct net_device *dev = ns->netdev; 989 int i; 990 991 for (i = 0; i < dev->num_rx_queues; i++) 992 nsim_queue_free(dev, ns->rq[i]); 993 994 kfree(ns->rq); 995 ns->rq = NULL; 996 } 997 998 static int nsim_init_netdevsim(struct netdevsim *ns) 999 { 1000 struct mock_phc *phc; 1001 int err; 1002 1003 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 1004 if (IS_ERR(phc)) 1005 return PTR_ERR(phc); 1006 1007 ns->phc = phc; 1008 ns->netdev->netdev_ops = &nsim_netdev_ops; 1009 ns->netdev->stat_ops = &nsim_stat_ops; 1010 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 1011 netdev_lockdep_set_classes(ns->netdev); 1012 1013 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 1014 if (err) 1015 goto err_phc_destroy; 1016 1017 rtnl_lock(); 1018 err = nsim_queue_init(ns); 1019 if (err) 1020 goto err_utn_destroy; 1021 1022 err = nsim_bpf_init(ns); 1023 if (err) 1024 goto err_rq_destroy; 1025 1026 nsim_macsec_init(ns); 1027 nsim_ipsec_init(ns); 1028 1029 err = register_netdevice(ns->netdev); 1030 if (err) 1031 goto err_ipsec_teardown; 1032 rtnl_unlock(); 1033 1034 if (IS_ENABLED(CONFIG_DEBUG_NET)) { 1035 ns->nb.notifier_call = netdev_debug_event; 1036 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1037 &ns->nn)) 1038 ns->nb.notifier_call = NULL; 1039 } 1040 1041 return 0; 1042 1043 err_ipsec_teardown: 1044 nsim_ipsec_teardown(ns); 1045 nsim_macsec_teardown(ns); 1046 nsim_bpf_uninit(ns); 1047 err_rq_destroy: 1048 nsim_queue_uninit(ns); 1049 err_utn_destroy: 1050 rtnl_unlock(); 1051 nsim_udp_tunnels_info_destroy(ns->netdev); 1052 err_phc_destroy: 1053 mock_phc_destroy(ns->phc); 1054 return err; 1055 } 1056 1057 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 1058 { 1059 int err; 1060 1061 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 1062 rtnl_lock(); 1063 err = register_netdevice(ns->netdev); 1064 rtnl_unlock(); 1065 return err; 1066 } 1067 1068 static void nsim_exit_netdevsim(struct netdevsim *ns) 1069 { 1070 nsim_udp_tunnels_info_destroy(ns->netdev); 1071 mock_phc_destroy(ns->phc); 1072 } 1073 1074 struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, 1075 struct nsim_dev_port *nsim_dev_port, 1076 u8 perm_addr[ETH_ALEN]) 1077 { 1078 struct net_device *dev; 1079 struct netdevsim *ns; 1080 int err; 1081 1082 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1083 nsim_dev->nsim_bus_dev->num_queues); 1084 if (!dev) 1085 return ERR_PTR(-ENOMEM); 1086 1087 if (perm_addr) 1088 memcpy(dev->perm_addr, perm_addr, ETH_ALEN); 1089 1090 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1091 ns = netdev_priv(dev); 1092 ns->netdev = dev; 1093 ns->nsim_dev = nsim_dev; 1094 ns->nsim_dev_port = nsim_dev_port; 1095 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1096 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1097 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1098 nsim_ethtool_init(ns); 1099 if (nsim_dev_port_is_pf(nsim_dev_port)) 1100 err = nsim_init_netdevsim(ns); 1101 else 1102 err = nsim_init_netdevsim_vf(ns); 1103 if (err) 1104 goto err_free_netdev; 1105 1106 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1107 ns, &nsim_pp_hold_fops); 1108 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1109 nsim_dev_port->ddir, ns, 1110 &nsim_qreset_fops); 1111 return ns; 1112 1113 err_free_netdev: 1114 free_netdev(dev); 1115 return ERR_PTR(err); 1116 } 1117 1118 void nsim_destroy(struct netdevsim *ns) 1119 { 1120 struct net_device *dev = ns->netdev; 1121 struct netdevsim *peer; 1122 1123 debugfs_remove(ns->qr_dfs); 1124 debugfs_remove(ns->pp_dfs); 1125 1126 if (ns->nb.notifier_call) 1127 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1128 &ns->nn); 1129 1130 rtnl_lock(); 1131 peer = rtnl_dereference(ns->peer); 1132 if (peer) 1133 RCU_INIT_POINTER(peer->peer, NULL); 1134 RCU_INIT_POINTER(ns->peer, NULL); 1135 unregister_netdevice(dev); 1136 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1137 nsim_macsec_teardown(ns); 1138 nsim_ipsec_teardown(ns); 1139 nsim_bpf_uninit(ns); 1140 nsim_queue_uninit(ns); 1141 } 1142 rtnl_unlock(); 1143 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1144 nsim_exit_netdevsim(ns); 1145 1146 /* Put this intentionally late to exercise the orphaning path */ 1147 if (ns->page) { 1148 page_pool_put_full_page(ns->page->pp, ns->page, false); 1149 ns->page = NULL; 1150 } 1151 1152 free_netdev(dev); 1153 } 1154 1155 bool netdev_is_nsim(struct net_device *dev) 1156 { 1157 return dev->netdev_ops == &nsim_netdev_ops; 1158 } 1159 1160 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1161 struct netlink_ext_ack *extack) 1162 { 1163 NL_SET_ERR_MSG_MOD(extack, 1164 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1165 return -EOPNOTSUPP; 1166 } 1167 1168 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1169 .kind = DRV_NAME, 1170 .validate = nsim_validate, 1171 }; 1172 1173 static int __init nsim_module_init(void) 1174 { 1175 int err; 1176 1177 err = nsim_dev_init(); 1178 if (err) 1179 return err; 1180 1181 err = nsim_bus_init(); 1182 if (err) 1183 goto err_dev_exit; 1184 1185 err = rtnl_link_register(&nsim_link_ops); 1186 if (err) 1187 goto err_bus_exit; 1188 1189 return 0; 1190 1191 err_bus_exit: 1192 nsim_bus_exit(); 1193 err_dev_exit: 1194 nsim_dev_exit(); 1195 return err; 1196 } 1197 1198 static void __exit nsim_module_exit(void) 1199 { 1200 rtnl_link_unregister(&nsim_link_ops); 1201 nsim_bus_exit(); 1202 nsim_dev_exit(); 1203 } 1204 1205 module_init(nsim_module_init); 1206 module_exit(nsim_module_exit); 1207 MODULE_LICENSE("GPL"); 1208 MODULE_DESCRIPTION("Simulated networking device for testing"); 1209 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1210