1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 33 #include "netdevsim.h" 34 35 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 36 37 #define NSIM_RING_SIZE 256 38 39 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb) 40 { 41 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 42 dev_kfree_skb_any(skb); 43 return NET_RX_DROP; 44 } 45 46 skb_queue_tail(&rq->skb_queue, skb); 47 return NET_RX_SUCCESS; 48 } 49 50 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb, 51 struct nsim_rq *rq) 52 { 53 return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb); 54 } 55 56 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 57 { 58 struct netdevsim *ns = netdev_priv(dev); 59 struct net_device *peer_dev; 60 unsigned int len = skb->len; 61 struct netdevsim *peer_ns; 62 struct netdev_config *cfg; 63 struct nsim_rq *rq; 64 int rxq; 65 66 rcu_read_lock(); 67 if (!nsim_ipsec_tx(ns, skb)) 68 goto out_drop_free; 69 70 peer_ns = rcu_dereference(ns->peer); 71 if (!peer_ns) 72 goto out_drop_free; 73 74 peer_dev = peer_ns->netdev; 75 rxq = skb_get_queue_mapping(skb); 76 if (rxq >= peer_dev->num_rx_queues) 77 rxq = rxq % peer_dev->num_rx_queues; 78 rq = peer_ns->rq[rxq]; 79 80 cfg = peer_dev->cfg; 81 if (skb_is_nonlinear(skb) && 82 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 83 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 84 cfg->hds_thresh > len))) 85 skb_linearize(skb); 86 87 skb_tx_timestamp(skb); 88 if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP)) 89 goto out_drop_cnt; 90 91 if (!hrtimer_active(&rq->napi_timer)) 92 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 93 94 rcu_read_unlock(); 95 u64_stats_update_begin(&ns->syncp); 96 ns->tx_packets++; 97 ns->tx_bytes += len; 98 u64_stats_update_end(&ns->syncp); 99 return NETDEV_TX_OK; 100 101 out_drop_free: 102 dev_kfree_skb(skb); 103 out_drop_cnt: 104 rcu_read_unlock(); 105 u64_stats_update_begin(&ns->syncp); 106 ns->tx_dropped++; 107 u64_stats_update_end(&ns->syncp); 108 return NETDEV_TX_OK; 109 } 110 111 static void nsim_set_rx_mode(struct net_device *dev) 112 { 113 } 114 115 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 116 { 117 struct netdevsim *ns = netdev_priv(dev); 118 119 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 120 new_mtu > NSIM_XDP_MAX_MTU) 121 return -EBUSY; 122 123 WRITE_ONCE(dev->mtu, new_mtu); 124 125 return 0; 126 } 127 128 static void 129 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 130 { 131 struct netdevsim *ns = netdev_priv(dev); 132 unsigned int start; 133 134 do { 135 start = u64_stats_fetch_begin(&ns->syncp); 136 stats->tx_bytes = ns->tx_bytes; 137 stats->tx_packets = ns->tx_packets; 138 stats->tx_dropped = ns->tx_dropped; 139 } while (u64_stats_fetch_retry(&ns->syncp, start)); 140 } 141 142 static int 143 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 144 { 145 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv); 146 } 147 148 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 149 { 150 struct netdevsim *ns = netdev_priv(dev); 151 struct nsim_dev *nsim_dev = ns->nsim_dev; 152 153 /* Only refuse multicast addresses, zero address can mean unset/any. */ 154 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 155 return -EINVAL; 156 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 157 158 return 0; 159 } 160 161 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 162 u16 vlan, u8 qos, __be16 vlan_proto) 163 { 164 struct netdevsim *ns = netdev_priv(dev); 165 struct nsim_dev *nsim_dev = ns->nsim_dev; 166 167 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 168 return -EINVAL; 169 170 nsim_dev->vfconfigs[vf].vlan = vlan; 171 nsim_dev->vfconfigs[vf].qos = qos; 172 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 173 174 return 0; 175 } 176 177 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 178 { 179 struct netdevsim *ns = netdev_priv(dev); 180 struct nsim_dev *nsim_dev = ns->nsim_dev; 181 182 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 183 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 184 return -EOPNOTSUPP; 185 } 186 187 if (vf >= nsim_dev_get_vfs(nsim_dev)) 188 return -EINVAL; 189 190 nsim_dev->vfconfigs[vf].min_tx_rate = min; 191 nsim_dev->vfconfigs[vf].max_tx_rate = max; 192 193 return 0; 194 } 195 196 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 197 { 198 struct netdevsim *ns = netdev_priv(dev); 199 struct nsim_dev *nsim_dev = ns->nsim_dev; 200 201 if (vf >= nsim_dev_get_vfs(nsim_dev)) 202 return -EINVAL; 203 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 204 205 return 0; 206 } 207 208 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 209 { 210 struct netdevsim *ns = netdev_priv(dev); 211 struct nsim_dev *nsim_dev = ns->nsim_dev; 212 213 if (vf >= nsim_dev_get_vfs(nsim_dev)) 214 return -EINVAL; 215 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 216 217 return 0; 218 } 219 220 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 221 { 222 struct netdevsim *ns = netdev_priv(dev); 223 struct nsim_dev *nsim_dev = ns->nsim_dev; 224 225 if (vf >= nsim_dev_get_vfs(nsim_dev)) 226 return -EINVAL; 227 nsim_dev->vfconfigs[vf].trusted = val; 228 229 return 0; 230 } 231 232 static int 233 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 234 { 235 struct netdevsim *ns = netdev_priv(dev); 236 struct nsim_dev *nsim_dev = ns->nsim_dev; 237 238 if (vf >= nsim_dev_get_vfs(nsim_dev)) 239 return -EINVAL; 240 241 ivi->vf = vf; 242 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 243 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 244 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 245 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 246 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 247 ivi->qos = nsim_dev->vfconfigs[vf].qos; 248 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 249 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 250 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 251 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 252 253 return 0; 254 } 255 256 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 257 { 258 struct netdevsim *ns = netdev_priv(dev); 259 struct nsim_dev *nsim_dev = ns->nsim_dev; 260 261 if (vf >= nsim_dev_get_vfs(nsim_dev)) 262 return -EINVAL; 263 264 switch (state) { 265 case IFLA_VF_LINK_STATE_AUTO: 266 case IFLA_VF_LINK_STATE_ENABLE: 267 case IFLA_VF_LINK_STATE_DISABLE: 268 break; 269 default: 270 return -EINVAL; 271 } 272 273 nsim_dev->vfconfigs[vf].link_state = state; 274 275 return 0; 276 } 277 278 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats) 279 { 280 stats->window_drops = 0; 281 stats->tx_overruns = 0; 282 } 283 284 static int nsim_setup_tc_taprio(struct net_device *dev, 285 struct tc_taprio_qopt_offload *offload) 286 { 287 int err = 0; 288 289 switch (offload->cmd) { 290 case TAPRIO_CMD_REPLACE: 291 case TAPRIO_CMD_DESTROY: 292 break; 293 case TAPRIO_CMD_STATS: 294 nsim_taprio_stats(&offload->stats); 295 break; 296 default: 297 err = -EOPNOTSUPP; 298 } 299 300 return err; 301 } 302 303 static LIST_HEAD(nsim_block_cb_list); 304 305 static int 306 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) 307 { 308 struct netdevsim *ns = netdev_priv(dev); 309 310 switch (type) { 311 case TC_SETUP_QDISC_TAPRIO: 312 return nsim_setup_tc_taprio(dev, type_data); 313 case TC_SETUP_BLOCK: 314 return flow_block_cb_setup_simple(type_data, 315 &nsim_block_cb_list, 316 nsim_setup_tc_block_cb, 317 ns, ns, true); 318 default: 319 return -EOPNOTSUPP; 320 } 321 } 322 323 static int 324 nsim_set_features(struct net_device *dev, netdev_features_t features) 325 { 326 struct netdevsim *ns = netdev_priv(dev); 327 328 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 329 return nsim_bpf_disable_tc(ns); 330 331 return 0; 332 } 333 334 static int nsim_get_iflink(const struct net_device *dev) 335 { 336 struct netdevsim *nsim, *peer; 337 int iflink; 338 339 nsim = netdev_priv(dev); 340 341 rcu_read_lock(); 342 peer = rcu_dereference(nsim->peer); 343 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 344 READ_ONCE(dev->ifindex); 345 rcu_read_unlock(); 346 347 return iflink; 348 } 349 350 static int nsim_rcv(struct nsim_rq *rq, int budget) 351 { 352 struct sk_buff *skb; 353 int i; 354 355 for (i = 0; i < budget; i++) { 356 if (skb_queue_empty(&rq->skb_queue)) 357 break; 358 359 skb = skb_dequeue(&rq->skb_queue); 360 netif_receive_skb(skb); 361 } 362 363 return i; 364 } 365 366 static int nsim_poll(struct napi_struct *napi, int budget) 367 { 368 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 369 int done; 370 371 done = nsim_rcv(rq, budget); 372 napi_complete(napi); 373 374 return done; 375 } 376 377 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 378 { 379 struct page_pool_params params = { 380 .order = 0, 381 .pool_size = NSIM_RING_SIZE, 382 .nid = NUMA_NO_NODE, 383 .dev = &napi->dev->dev, 384 .napi = napi, 385 .dma_dir = DMA_BIDIRECTIONAL, 386 .netdev = napi->dev, 387 }; 388 struct page_pool *pool; 389 390 pool = page_pool_create(¶ms); 391 if (IS_ERR(pool)) 392 return PTR_ERR(pool); 393 394 *p = pool; 395 return 0; 396 } 397 398 static int nsim_init_napi(struct netdevsim *ns) 399 { 400 struct net_device *dev = ns->netdev; 401 struct nsim_rq *rq; 402 int err, i; 403 404 for (i = 0; i < dev->num_rx_queues; i++) { 405 rq = ns->rq[i]; 406 407 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 408 } 409 410 for (i = 0; i < dev->num_rx_queues; i++) { 411 rq = ns->rq[i]; 412 413 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 414 if (err) 415 goto err_pp_destroy; 416 } 417 418 return 0; 419 420 err_pp_destroy: 421 while (i--) { 422 page_pool_destroy(ns->rq[i]->page_pool); 423 ns->rq[i]->page_pool = NULL; 424 } 425 426 for (i = 0; i < dev->num_rx_queues; i++) 427 __netif_napi_del_locked(&ns->rq[i]->napi); 428 429 return err; 430 } 431 432 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 433 { 434 struct nsim_rq *rq; 435 436 rq = container_of(timer, struct nsim_rq, napi_timer); 437 napi_schedule(&rq->napi); 438 439 return HRTIMER_NORESTART; 440 } 441 442 static void nsim_rq_timer_init(struct nsim_rq *rq) 443 { 444 hrtimer_init(&rq->napi_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 445 rq->napi_timer.function = nsim_napi_schedule; 446 } 447 448 static void nsim_enable_napi(struct netdevsim *ns) 449 { 450 struct net_device *dev = ns->netdev; 451 int i; 452 453 for (i = 0; i < dev->num_rx_queues; i++) { 454 struct nsim_rq *rq = ns->rq[i]; 455 456 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 457 napi_enable_locked(&rq->napi); 458 } 459 } 460 461 static int nsim_open(struct net_device *dev) 462 { 463 struct netdevsim *ns = netdev_priv(dev); 464 int err; 465 466 netdev_assert_locked(dev); 467 468 err = nsim_init_napi(ns); 469 if (err) 470 return err; 471 472 nsim_enable_napi(ns); 473 474 return 0; 475 } 476 477 static void nsim_del_napi(struct netdevsim *ns) 478 { 479 struct net_device *dev = ns->netdev; 480 int i; 481 482 for (i = 0; i < dev->num_rx_queues; i++) { 483 struct nsim_rq *rq = ns->rq[i]; 484 485 napi_disable_locked(&rq->napi); 486 __netif_napi_del_locked(&rq->napi); 487 } 488 synchronize_net(); 489 490 for (i = 0; i < dev->num_rx_queues; i++) { 491 page_pool_destroy(ns->rq[i]->page_pool); 492 ns->rq[i]->page_pool = NULL; 493 } 494 } 495 496 static int nsim_stop(struct net_device *dev) 497 { 498 struct netdevsim *ns = netdev_priv(dev); 499 struct netdevsim *peer; 500 501 netdev_assert_locked(dev); 502 503 netif_carrier_off(dev); 504 peer = rtnl_dereference(ns->peer); 505 if (peer) 506 netif_carrier_off(peer->netdev); 507 508 nsim_del_napi(ns); 509 510 return 0; 511 } 512 513 static int nsim_shaper_set(struct net_shaper_binding *binding, 514 const struct net_shaper *shaper, 515 struct netlink_ext_ack *extack) 516 { 517 return 0; 518 } 519 520 static int nsim_shaper_del(struct net_shaper_binding *binding, 521 const struct net_shaper_handle *handle, 522 struct netlink_ext_ack *extack) 523 { 524 return 0; 525 } 526 527 static int nsim_shaper_group(struct net_shaper_binding *binding, 528 int leaves_count, 529 const struct net_shaper *leaves, 530 const struct net_shaper *root, 531 struct netlink_ext_ack *extack) 532 { 533 return 0; 534 } 535 536 static void nsim_shaper_cap(struct net_shaper_binding *binding, 537 enum net_shaper_scope scope, 538 unsigned long *flags) 539 { 540 *flags = ULONG_MAX; 541 } 542 543 static const struct net_shaper_ops nsim_shaper_ops = { 544 .set = nsim_shaper_set, 545 .delete = nsim_shaper_del, 546 .group = nsim_shaper_group, 547 .capabilities = nsim_shaper_cap, 548 }; 549 550 static const struct net_device_ops nsim_netdev_ops = { 551 .ndo_start_xmit = nsim_start_xmit, 552 .ndo_set_rx_mode = nsim_set_rx_mode, 553 .ndo_set_mac_address = eth_mac_addr, 554 .ndo_validate_addr = eth_validate_addr, 555 .ndo_change_mtu = nsim_change_mtu, 556 .ndo_get_stats64 = nsim_get_stats64, 557 .ndo_set_vf_mac = nsim_set_vf_mac, 558 .ndo_set_vf_vlan = nsim_set_vf_vlan, 559 .ndo_set_vf_rate = nsim_set_vf_rate, 560 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 561 .ndo_set_vf_trust = nsim_set_vf_trust, 562 .ndo_get_vf_config = nsim_get_vf_config, 563 .ndo_set_vf_link_state = nsim_set_vf_link_state, 564 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 565 .ndo_setup_tc = nsim_setup_tc, 566 .ndo_set_features = nsim_set_features, 567 .ndo_get_iflink = nsim_get_iflink, 568 .ndo_bpf = nsim_bpf, 569 .ndo_open = nsim_open, 570 .ndo_stop = nsim_stop, 571 .net_shaper_ops = &nsim_shaper_ops, 572 }; 573 574 static const struct net_device_ops nsim_vf_netdev_ops = { 575 .ndo_start_xmit = nsim_start_xmit, 576 .ndo_set_rx_mode = nsim_set_rx_mode, 577 .ndo_set_mac_address = eth_mac_addr, 578 .ndo_validate_addr = eth_validate_addr, 579 .ndo_change_mtu = nsim_change_mtu, 580 .ndo_get_stats64 = nsim_get_stats64, 581 .ndo_setup_tc = nsim_setup_tc, 582 .ndo_set_features = nsim_set_features, 583 }; 584 585 /* We don't have true per-queue stats, yet, so do some random fakery here. 586 * Only report stuff for queue 0. 587 */ 588 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 589 struct netdev_queue_stats_rx *stats) 590 { 591 struct rtnl_link_stats64 rtstats = {}; 592 593 if (!idx) 594 nsim_get_stats64(dev, &rtstats); 595 596 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 597 stats->bytes = rtstats.rx_bytes; 598 } 599 600 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 601 struct netdev_queue_stats_tx *stats) 602 { 603 struct rtnl_link_stats64 rtstats = {}; 604 605 if (!idx) 606 nsim_get_stats64(dev, &rtstats); 607 608 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 609 stats->bytes = rtstats.tx_bytes; 610 } 611 612 static void nsim_get_base_stats(struct net_device *dev, 613 struct netdev_queue_stats_rx *rx, 614 struct netdev_queue_stats_tx *tx) 615 { 616 struct rtnl_link_stats64 rtstats = {}; 617 618 nsim_get_stats64(dev, &rtstats); 619 620 rx->packets = !!rtstats.rx_packets; 621 rx->bytes = 0; 622 tx->packets = !!rtstats.tx_packets; 623 tx->bytes = 0; 624 } 625 626 static const struct netdev_stat_ops nsim_stat_ops = { 627 .get_queue_stats_tx = nsim_get_queue_stats_tx, 628 .get_queue_stats_rx = nsim_get_queue_stats_rx, 629 .get_base_stats = nsim_get_base_stats, 630 }; 631 632 static struct nsim_rq *nsim_queue_alloc(void) 633 { 634 struct nsim_rq *rq; 635 636 rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT); 637 if (!rq) 638 return NULL; 639 640 skb_queue_head_init(&rq->skb_queue); 641 nsim_rq_timer_init(rq); 642 return rq; 643 } 644 645 static void nsim_queue_free(struct nsim_rq *rq) 646 { 647 hrtimer_cancel(&rq->napi_timer); 648 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 649 kfree(rq); 650 } 651 652 /* Queue reset mode is controlled by ns->rq_reset_mode. 653 * - normal - new NAPI new pool (old NAPI enabled when new added) 654 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 655 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 656 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 657 */ 658 struct nsim_queue_mem { 659 struct nsim_rq *rq; 660 struct page_pool *pp; 661 }; 662 663 static int 664 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx) 665 { 666 struct nsim_queue_mem *qmem = per_queue_mem; 667 struct netdevsim *ns = netdev_priv(dev); 668 int err; 669 670 if (ns->rq_reset_mode > 3) 671 return -EINVAL; 672 673 if (ns->rq_reset_mode == 1) { 674 if (!netif_running(ns->netdev)) 675 return -ENETDOWN; 676 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 677 } 678 679 qmem->rq = nsim_queue_alloc(); 680 if (!qmem->rq) 681 return -ENOMEM; 682 683 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 684 if (err) 685 goto err_free; 686 687 if (!ns->rq_reset_mode) 688 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 689 idx); 690 691 return 0; 692 693 err_free: 694 nsim_queue_free(qmem->rq); 695 return err; 696 } 697 698 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 699 { 700 struct nsim_queue_mem *qmem = per_queue_mem; 701 struct netdevsim *ns = netdev_priv(dev); 702 703 page_pool_destroy(qmem->pp); 704 if (qmem->rq) { 705 if (!ns->rq_reset_mode) 706 netif_napi_del_locked(&qmem->rq->napi); 707 page_pool_destroy(qmem->rq->page_pool); 708 nsim_queue_free(qmem->rq); 709 } 710 } 711 712 static int 713 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) 714 { 715 struct nsim_queue_mem *qmem = per_queue_mem; 716 struct netdevsim *ns = netdev_priv(dev); 717 718 netdev_assert_locked(dev); 719 720 if (ns->rq_reset_mode == 1) { 721 ns->rq[idx]->page_pool = qmem->pp; 722 napi_enable_locked(&ns->rq[idx]->napi); 723 return 0; 724 } 725 726 /* netif_napi_add()/_del() should normally be called from alloc/free, 727 * here we want to test various call orders. 728 */ 729 if (ns->rq_reset_mode == 2) { 730 netif_napi_del_locked(&ns->rq[idx]->napi); 731 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 732 idx); 733 } else if (ns->rq_reset_mode == 3) { 734 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 735 idx); 736 netif_napi_del_locked(&ns->rq[idx]->napi); 737 } 738 739 ns->rq[idx] = qmem->rq; 740 napi_enable_locked(&ns->rq[idx]->napi); 741 742 return 0; 743 } 744 745 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 746 { 747 struct nsim_queue_mem *qmem = per_queue_mem; 748 struct netdevsim *ns = netdev_priv(dev); 749 750 netdev_assert_locked(dev); 751 752 napi_disable_locked(&ns->rq[idx]->napi); 753 754 if (ns->rq_reset_mode == 1) { 755 qmem->pp = ns->rq[idx]->page_pool; 756 page_pool_disable_direct_recycling(qmem->pp); 757 } else { 758 qmem->rq = ns->rq[idx]; 759 } 760 761 return 0; 762 } 763 764 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 765 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 766 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 767 .ndo_queue_mem_free = nsim_queue_mem_free, 768 .ndo_queue_start = nsim_queue_start, 769 .ndo_queue_stop = nsim_queue_stop, 770 }; 771 772 static ssize_t 773 nsim_qreset_write(struct file *file, const char __user *data, 774 size_t count, loff_t *ppos) 775 { 776 struct netdevsim *ns = file->private_data; 777 unsigned int queue, mode; 778 char buf[32]; 779 ssize_t ret; 780 781 if (count >= sizeof(buf)) 782 return -EINVAL; 783 if (copy_from_user(buf, data, count)) 784 return -EFAULT; 785 buf[count] = '\0'; 786 787 ret = sscanf(buf, "%u %u", &queue, &mode); 788 if (ret != 2) 789 return -EINVAL; 790 791 netdev_lock(ns->netdev); 792 if (queue >= ns->netdev->real_num_rx_queues) { 793 ret = -EINVAL; 794 goto exit_unlock; 795 } 796 797 ns->rq_reset_mode = mode; 798 ret = netdev_rx_queue_restart(ns->netdev, queue); 799 ns->rq_reset_mode = 0; 800 if (ret) 801 goto exit_unlock; 802 803 ret = count; 804 exit_unlock: 805 netdev_unlock(ns->netdev); 806 return ret; 807 } 808 809 static const struct file_operations nsim_qreset_fops = { 810 .open = simple_open, 811 .write = nsim_qreset_write, 812 .owner = THIS_MODULE, 813 }; 814 815 static ssize_t 816 nsim_pp_hold_read(struct file *file, char __user *data, 817 size_t count, loff_t *ppos) 818 { 819 struct netdevsim *ns = file->private_data; 820 char buf[3] = "n\n"; 821 822 if (ns->page) 823 buf[0] = 'y'; 824 825 return simple_read_from_buffer(data, count, ppos, buf, 2); 826 } 827 828 static ssize_t 829 nsim_pp_hold_write(struct file *file, const char __user *data, 830 size_t count, loff_t *ppos) 831 { 832 struct netdevsim *ns = file->private_data; 833 ssize_t ret; 834 bool val; 835 836 ret = kstrtobool_from_user(data, count, &val); 837 if (ret) 838 return ret; 839 840 rtnl_lock(); 841 ret = count; 842 if (val == !!ns->page) 843 goto exit; 844 845 if (!netif_running(ns->netdev) && val) { 846 ret = -ENETDOWN; 847 } else if (val) { 848 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 849 if (!ns->page) 850 ret = -ENOMEM; 851 } else { 852 page_pool_put_full_page(ns->page->pp, ns->page, false); 853 ns->page = NULL; 854 } 855 856 exit: 857 rtnl_unlock(); 858 return ret; 859 } 860 861 static const struct file_operations nsim_pp_hold_fops = { 862 .open = simple_open, 863 .read = nsim_pp_hold_read, 864 .write = nsim_pp_hold_write, 865 .llseek = generic_file_llseek, 866 .owner = THIS_MODULE, 867 }; 868 869 static void nsim_setup(struct net_device *dev) 870 { 871 ether_setup(dev); 872 eth_hw_addr_random(dev); 873 874 dev->tx_queue_len = 0; 875 dev->flags &= ~IFF_MULTICAST; 876 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | 877 IFF_NO_QUEUE; 878 dev->features |= NETIF_F_HIGHDMA | 879 NETIF_F_SG | 880 NETIF_F_FRAGLIST | 881 NETIF_F_HW_CSUM | 882 NETIF_F_TSO; 883 dev->hw_features |= NETIF_F_HW_TC | 884 NETIF_F_SG | 885 NETIF_F_FRAGLIST | 886 NETIF_F_HW_CSUM | 887 NETIF_F_TSO; 888 dev->max_mtu = ETH_MAX_MTU; 889 dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD; 890 } 891 892 static int nsim_queue_init(struct netdevsim *ns) 893 { 894 struct net_device *dev = ns->netdev; 895 int i; 896 897 ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq), 898 GFP_KERNEL_ACCOUNT); 899 if (!ns->rq) 900 return -ENOMEM; 901 902 for (i = 0; i < dev->num_rx_queues; i++) { 903 ns->rq[i] = nsim_queue_alloc(); 904 if (!ns->rq[i]) 905 goto err_free_prev; 906 } 907 908 return 0; 909 910 err_free_prev: 911 while (i--) 912 kfree(ns->rq[i]); 913 kfree(ns->rq); 914 return -ENOMEM; 915 } 916 917 static void nsim_queue_uninit(struct netdevsim *ns) 918 { 919 struct net_device *dev = ns->netdev; 920 int i; 921 922 for (i = 0; i < dev->num_rx_queues; i++) 923 nsim_queue_free(ns->rq[i]); 924 925 kfree(ns->rq); 926 ns->rq = NULL; 927 } 928 929 static int nsim_init_netdevsim(struct netdevsim *ns) 930 { 931 struct mock_phc *phc; 932 int err; 933 934 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 935 if (IS_ERR(phc)) 936 return PTR_ERR(phc); 937 938 ns->phc = phc; 939 ns->netdev->netdev_ops = &nsim_netdev_ops; 940 ns->netdev->stat_ops = &nsim_stat_ops; 941 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 942 943 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 944 if (err) 945 goto err_phc_destroy; 946 947 rtnl_lock(); 948 err = nsim_queue_init(ns); 949 if (err) 950 goto err_utn_destroy; 951 952 err = nsim_bpf_init(ns); 953 if (err) 954 goto err_rq_destroy; 955 956 nsim_macsec_init(ns); 957 nsim_ipsec_init(ns); 958 959 err = register_netdevice(ns->netdev); 960 if (err) 961 goto err_ipsec_teardown; 962 rtnl_unlock(); 963 return 0; 964 965 err_ipsec_teardown: 966 nsim_ipsec_teardown(ns); 967 nsim_macsec_teardown(ns); 968 nsim_bpf_uninit(ns); 969 err_rq_destroy: 970 nsim_queue_uninit(ns); 971 err_utn_destroy: 972 rtnl_unlock(); 973 nsim_udp_tunnels_info_destroy(ns->netdev); 974 err_phc_destroy: 975 mock_phc_destroy(ns->phc); 976 return err; 977 } 978 979 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 980 { 981 int err; 982 983 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 984 rtnl_lock(); 985 err = register_netdevice(ns->netdev); 986 rtnl_unlock(); 987 return err; 988 } 989 990 static void nsim_exit_netdevsim(struct netdevsim *ns) 991 { 992 nsim_udp_tunnels_info_destroy(ns->netdev); 993 mock_phc_destroy(ns->phc); 994 } 995 996 struct netdevsim * 997 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port) 998 { 999 struct net_device *dev; 1000 struct netdevsim *ns; 1001 int err; 1002 1003 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1004 nsim_dev->nsim_bus_dev->num_queues); 1005 if (!dev) 1006 return ERR_PTR(-ENOMEM); 1007 1008 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1009 ns = netdev_priv(dev); 1010 ns->netdev = dev; 1011 u64_stats_init(&ns->syncp); 1012 ns->nsim_dev = nsim_dev; 1013 ns->nsim_dev_port = nsim_dev_port; 1014 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1015 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1016 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1017 nsim_ethtool_init(ns); 1018 if (nsim_dev_port_is_pf(nsim_dev_port)) 1019 err = nsim_init_netdevsim(ns); 1020 else 1021 err = nsim_init_netdevsim_vf(ns); 1022 if (err) 1023 goto err_free_netdev; 1024 1025 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1026 ns, &nsim_pp_hold_fops); 1027 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1028 nsim_dev_port->ddir, ns, 1029 &nsim_qreset_fops); 1030 1031 return ns; 1032 1033 err_free_netdev: 1034 free_netdev(dev); 1035 return ERR_PTR(err); 1036 } 1037 1038 void nsim_destroy(struct netdevsim *ns) 1039 { 1040 struct net_device *dev = ns->netdev; 1041 struct netdevsim *peer; 1042 1043 debugfs_remove(ns->qr_dfs); 1044 debugfs_remove(ns->pp_dfs); 1045 1046 rtnl_lock(); 1047 peer = rtnl_dereference(ns->peer); 1048 if (peer) 1049 RCU_INIT_POINTER(peer->peer, NULL); 1050 RCU_INIT_POINTER(ns->peer, NULL); 1051 unregister_netdevice(dev); 1052 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1053 nsim_macsec_teardown(ns); 1054 nsim_ipsec_teardown(ns); 1055 nsim_bpf_uninit(ns); 1056 nsim_queue_uninit(ns); 1057 } 1058 rtnl_unlock(); 1059 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1060 nsim_exit_netdevsim(ns); 1061 1062 /* Put this intentionally late to exercise the orphaning path */ 1063 if (ns->page) { 1064 page_pool_put_full_page(ns->page->pp, ns->page, false); 1065 ns->page = NULL; 1066 } 1067 1068 free_netdev(dev); 1069 } 1070 1071 bool netdev_is_nsim(struct net_device *dev) 1072 { 1073 return dev->netdev_ops == &nsim_netdev_ops; 1074 } 1075 1076 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1077 struct netlink_ext_ack *extack) 1078 { 1079 NL_SET_ERR_MSG_MOD(extack, 1080 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1081 return -EOPNOTSUPP; 1082 } 1083 1084 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1085 .kind = DRV_NAME, 1086 .validate = nsim_validate, 1087 }; 1088 1089 static int __init nsim_module_init(void) 1090 { 1091 int err; 1092 1093 err = nsim_dev_init(); 1094 if (err) 1095 return err; 1096 1097 err = nsim_bus_init(); 1098 if (err) 1099 goto err_dev_exit; 1100 1101 err = rtnl_link_register(&nsim_link_ops); 1102 if (err) 1103 goto err_bus_exit; 1104 1105 return 0; 1106 1107 err_bus_exit: 1108 nsim_bus_exit(); 1109 err_dev_exit: 1110 nsim_dev_exit(); 1111 return err; 1112 } 1113 1114 static void __exit nsim_module_exit(void) 1115 { 1116 rtnl_link_unregister(&nsim_link_ops); 1117 nsim_bus_exit(); 1118 nsim_dev_exit(); 1119 } 1120 1121 module_init(nsim_module_init); 1122 module_exit(nsim_module_exit); 1123 MODULE_LICENSE("GPL"); 1124 MODULE_DESCRIPTION("Simulated networking device for testing"); 1125 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1126