1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/pkt_cls.h> 29 #include <net/rtnetlink.h> 30 #include <net/udp_tunnel.h> 31 32 #include "netdevsim.h" 33 34 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 35 36 #define NSIM_RING_SIZE 256 37 38 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb) 39 { 40 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 41 dev_kfree_skb_any(skb); 42 return NET_RX_DROP; 43 } 44 45 skb_queue_tail(&rq->skb_queue, skb); 46 return NET_RX_SUCCESS; 47 } 48 49 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb, 50 struct nsim_rq *rq) 51 { 52 return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb); 53 } 54 55 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 56 { 57 struct netdevsim *ns = netdev_priv(dev); 58 struct net_device *peer_dev; 59 unsigned int len = skb->len; 60 struct netdevsim *peer_ns; 61 struct netdev_config *cfg; 62 struct nsim_rq *rq; 63 int rxq; 64 65 rcu_read_lock(); 66 if (!nsim_ipsec_tx(ns, skb)) 67 goto out_drop_free; 68 69 peer_ns = rcu_dereference(ns->peer); 70 if (!peer_ns) 71 goto out_drop_free; 72 73 peer_dev = peer_ns->netdev; 74 rxq = skb_get_queue_mapping(skb); 75 if (rxq >= peer_dev->num_rx_queues) 76 rxq = rxq % peer_dev->num_rx_queues; 77 rq = peer_ns->rq[rxq]; 78 79 cfg = peer_dev->cfg; 80 if (skb_is_nonlinear(skb) && 81 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 82 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 83 cfg->hds_thresh > len))) 84 skb_linearize(skb); 85 86 skb_tx_timestamp(skb); 87 if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP)) 88 goto out_drop_cnt; 89 90 if (!hrtimer_active(&rq->napi_timer)) 91 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 92 93 rcu_read_unlock(); 94 u64_stats_update_begin(&ns->syncp); 95 ns->tx_packets++; 96 ns->tx_bytes += len; 97 u64_stats_update_end(&ns->syncp); 98 return NETDEV_TX_OK; 99 100 out_drop_free: 101 dev_kfree_skb(skb); 102 out_drop_cnt: 103 rcu_read_unlock(); 104 u64_stats_update_begin(&ns->syncp); 105 ns->tx_dropped++; 106 u64_stats_update_end(&ns->syncp); 107 return NETDEV_TX_OK; 108 } 109 110 static void nsim_set_rx_mode(struct net_device *dev) 111 { 112 } 113 114 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 115 { 116 struct netdevsim *ns = netdev_priv(dev); 117 118 if (ns->xdp.prog && new_mtu > NSIM_XDP_MAX_MTU) 119 return -EBUSY; 120 121 WRITE_ONCE(dev->mtu, new_mtu); 122 123 return 0; 124 } 125 126 static void 127 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 128 { 129 struct netdevsim *ns = netdev_priv(dev); 130 unsigned int start; 131 132 do { 133 start = u64_stats_fetch_begin(&ns->syncp); 134 stats->tx_bytes = ns->tx_bytes; 135 stats->tx_packets = ns->tx_packets; 136 stats->tx_dropped = ns->tx_dropped; 137 } while (u64_stats_fetch_retry(&ns->syncp, start)); 138 } 139 140 static int 141 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 142 { 143 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv); 144 } 145 146 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 147 { 148 struct netdevsim *ns = netdev_priv(dev); 149 struct nsim_dev *nsim_dev = ns->nsim_dev; 150 151 /* Only refuse multicast addresses, zero address can mean unset/any. */ 152 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 153 return -EINVAL; 154 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 155 156 return 0; 157 } 158 159 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 160 u16 vlan, u8 qos, __be16 vlan_proto) 161 { 162 struct netdevsim *ns = netdev_priv(dev); 163 struct nsim_dev *nsim_dev = ns->nsim_dev; 164 165 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 166 return -EINVAL; 167 168 nsim_dev->vfconfigs[vf].vlan = vlan; 169 nsim_dev->vfconfigs[vf].qos = qos; 170 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 171 172 return 0; 173 } 174 175 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 176 { 177 struct netdevsim *ns = netdev_priv(dev); 178 struct nsim_dev *nsim_dev = ns->nsim_dev; 179 180 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 181 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 182 return -EOPNOTSUPP; 183 } 184 185 if (vf >= nsim_dev_get_vfs(nsim_dev)) 186 return -EINVAL; 187 188 nsim_dev->vfconfigs[vf].min_tx_rate = min; 189 nsim_dev->vfconfigs[vf].max_tx_rate = max; 190 191 return 0; 192 } 193 194 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 195 { 196 struct netdevsim *ns = netdev_priv(dev); 197 struct nsim_dev *nsim_dev = ns->nsim_dev; 198 199 if (vf >= nsim_dev_get_vfs(nsim_dev)) 200 return -EINVAL; 201 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 202 203 return 0; 204 } 205 206 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 207 { 208 struct netdevsim *ns = netdev_priv(dev); 209 struct nsim_dev *nsim_dev = ns->nsim_dev; 210 211 if (vf >= nsim_dev_get_vfs(nsim_dev)) 212 return -EINVAL; 213 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 214 215 return 0; 216 } 217 218 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 219 { 220 struct netdevsim *ns = netdev_priv(dev); 221 struct nsim_dev *nsim_dev = ns->nsim_dev; 222 223 if (vf >= nsim_dev_get_vfs(nsim_dev)) 224 return -EINVAL; 225 nsim_dev->vfconfigs[vf].trusted = val; 226 227 return 0; 228 } 229 230 static int 231 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 232 { 233 struct netdevsim *ns = netdev_priv(dev); 234 struct nsim_dev *nsim_dev = ns->nsim_dev; 235 236 if (vf >= nsim_dev_get_vfs(nsim_dev)) 237 return -EINVAL; 238 239 ivi->vf = vf; 240 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 241 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 242 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 243 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 244 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 245 ivi->qos = nsim_dev->vfconfigs[vf].qos; 246 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 247 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 248 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 249 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 250 251 return 0; 252 } 253 254 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 255 { 256 struct netdevsim *ns = netdev_priv(dev); 257 struct nsim_dev *nsim_dev = ns->nsim_dev; 258 259 if (vf >= nsim_dev_get_vfs(nsim_dev)) 260 return -EINVAL; 261 262 switch (state) { 263 case IFLA_VF_LINK_STATE_AUTO: 264 case IFLA_VF_LINK_STATE_ENABLE: 265 case IFLA_VF_LINK_STATE_DISABLE: 266 break; 267 default: 268 return -EINVAL; 269 } 270 271 nsim_dev->vfconfigs[vf].link_state = state; 272 273 return 0; 274 } 275 276 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats) 277 { 278 stats->window_drops = 0; 279 stats->tx_overruns = 0; 280 } 281 282 static int nsim_setup_tc_taprio(struct net_device *dev, 283 struct tc_taprio_qopt_offload *offload) 284 { 285 int err = 0; 286 287 switch (offload->cmd) { 288 case TAPRIO_CMD_REPLACE: 289 case TAPRIO_CMD_DESTROY: 290 break; 291 case TAPRIO_CMD_STATS: 292 nsim_taprio_stats(&offload->stats); 293 break; 294 default: 295 err = -EOPNOTSUPP; 296 } 297 298 return err; 299 } 300 301 static LIST_HEAD(nsim_block_cb_list); 302 303 static int 304 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) 305 { 306 struct netdevsim *ns = netdev_priv(dev); 307 308 switch (type) { 309 case TC_SETUP_QDISC_TAPRIO: 310 return nsim_setup_tc_taprio(dev, type_data); 311 case TC_SETUP_BLOCK: 312 return flow_block_cb_setup_simple(type_data, 313 &nsim_block_cb_list, 314 nsim_setup_tc_block_cb, 315 ns, ns, true); 316 default: 317 return -EOPNOTSUPP; 318 } 319 } 320 321 static int 322 nsim_set_features(struct net_device *dev, netdev_features_t features) 323 { 324 struct netdevsim *ns = netdev_priv(dev); 325 326 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 327 return nsim_bpf_disable_tc(ns); 328 329 return 0; 330 } 331 332 static int nsim_get_iflink(const struct net_device *dev) 333 { 334 struct netdevsim *nsim, *peer; 335 int iflink; 336 337 nsim = netdev_priv(dev); 338 339 rcu_read_lock(); 340 peer = rcu_dereference(nsim->peer); 341 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 342 READ_ONCE(dev->ifindex); 343 rcu_read_unlock(); 344 345 return iflink; 346 } 347 348 static int nsim_rcv(struct nsim_rq *rq, int budget) 349 { 350 struct sk_buff *skb; 351 int i; 352 353 for (i = 0; i < budget; i++) { 354 if (skb_queue_empty(&rq->skb_queue)) 355 break; 356 357 skb = skb_dequeue(&rq->skb_queue); 358 netif_receive_skb(skb); 359 } 360 361 return i; 362 } 363 364 static int nsim_poll(struct napi_struct *napi, int budget) 365 { 366 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 367 int done; 368 369 done = nsim_rcv(rq, budget); 370 napi_complete(napi); 371 372 return done; 373 } 374 375 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 376 { 377 struct page_pool_params params = { 378 .order = 0, 379 .pool_size = NSIM_RING_SIZE, 380 .nid = NUMA_NO_NODE, 381 .dev = &napi->dev->dev, 382 .napi = napi, 383 .dma_dir = DMA_BIDIRECTIONAL, 384 .netdev = napi->dev, 385 }; 386 struct page_pool *pool; 387 388 pool = page_pool_create(¶ms); 389 if (IS_ERR(pool)) 390 return PTR_ERR(pool); 391 392 *p = pool; 393 return 0; 394 } 395 396 static int nsim_init_napi(struct netdevsim *ns) 397 { 398 struct net_device *dev = ns->netdev; 399 struct nsim_rq *rq; 400 int err, i; 401 402 for (i = 0; i < dev->num_rx_queues; i++) { 403 rq = ns->rq[i]; 404 405 netif_napi_add_config(dev, &rq->napi, nsim_poll, i); 406 } 407 408 for (i = 0; i < dev->num_rx_queues; i++) { 409 rq = ns->rq[i]; 410 411 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 412 if (err) 413 goto err_pp_destroy; 414 } 415 416 return 0; 417 418 err_pp_destroy: 419 while (i--) { 420 page_pool_destroy(ns->rq[i]->page_pool); 421 ns->rq[i]->page_pool = NULL; 422 } 423 424 for (i = 0; i < dev->num_rx_queues; i++) 425 __netif_napi_del(&ns->rq[i]->napi); 426 427 return err; 428 } 429 430 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 431 { 432 struct nsim_rq *rq; 433 434 rq = container_of(timer, struct nsim_rq, napi_timer); 435 napi_schedule(&rq->napi); 436 437 return HRTIMER_NORESTART; 438 } 439 440 static void nsim_rq_timer_init(struct nsim_rq *rq) 441 { 442 hrtimer_init(&rq->napi_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 443 rq->napi_timer.function = nsim_napi_schedule; 444 } 445 446 static void nsim_enable_napi(struct netdevsim *ns) 447 { 448 struct net_device *dev = ns->netdev; 449 int i; 450 451 for (i = 0; i < dev->num_rx_queues; i++) { 452 struct nsim_rq *rq = ns->rq[i]; 453 454 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 455 napi_enable(&rq->napi); 456 } 457 } 458 459 static int nsim_open(struct net_device *dev) 460 { 461 struct netdevsim *ns = netdev_priv(dev); 462 int err; 463 464 err = nsim_init_napi(ns); 465 if (err) 466 return err; 467 468 nsim_enable_napi(ns); 469 470 return 0; 471 } 472 473 static void nsim_del_napi(struct netdevsim *ns) 474 { 475 struct net_device *dev = ns->netdev; 476 int i; 477 478 for (i = 0; i < dev->num_rx_queues; i++) { 479 struct nsim_rq *rq = ns->rq[i]; 480 481 napi_disable(&rq->napi); 482 __netif_napi_del(&rq->napi); 483 } 484 synchronize_net(); 485 486 for (i = 0; i < dev->num_rx_queues; i++) { 487 page_pool_destroy(ns->rq[i]->page_pool); 488 ns->rq[i]->page_pool = NULL; 489 } 490 } 491 492 static int nsim_stop(struct net_device *dev) 493 { 494 struct netdevsim *ns = netdev_priv(dev); 495 struct netdevsim *peer; 496 497 netif_carrier_off(dev); 498 peer = rtnl_dereference(ns->peer); 499 if (peer) 500 netif_carrier_off(peer->netdev); 501 502 nsim_del_napi(ns); 503 504 return 0; 505 } 506 507 static int nsim_shaper_set(struct net_shaper_binding *binding, 508 const struct net_shaper *shaper, 509 struct netlink_ext_ack *extack) 510 { 511 return 0; 512 } 513 514 static int nsim_shaper_del(struct net_shaper_binding *binding, 515 const struct net_shaper_handle *handle, 516 struct netlink_ext_ack *extack) 517 { 518 return 0; 519 } 520 521 static int nsim_shaper_group(struct net_shaper_binding *binding, 522 int leaves_count, 523 const struct net_shaper *leaves, 524 const struct net_shaper *root, 525 struct netlink_ext_ack *extack) 526 { 527 return 0; 528 } 529 530 static void nsim_shaper_cap(struct net_shaper_binding *binding, 531 enum net_shaper_scope scope, 532 unsigned long *flags) 533 { 534 *flags = ULONG_MAX; 535 } 536 537 static const struct net_shaper_ops nsim_shaper_ops = { 538 .set = nsim_shaper_set, 539 .delete = nsim_shaper_del, 540 .group = nsim_shaper_group, 541 .capabilities = nsim_shaper_cap, 542 }; 543 544 static const struct net_device_ops nsim_netdev_ops = { 545 .ndo_start_xmit = nsim_start_xmit, 546 .ndo_set_rx_mode = nsim_set_rx_mode, 547 .ndo_set_mac_address = eth_mac_addr, 548 .ndo_validate_addr = eth_validate_addr, 549 .ndo_change_mtu = nsim_change_mtu, 550 .ndo_get_stats64 = nsim_get_stats64, 551 .ndo_set_vf_mac = nsim_set_vf_mac, 552 .ndo_set_vf_vlan = nsim_set_vf_vlan, 553 .ndo_set_vf_rate = nsim_set_vf_rate, 554 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 555 .ndo_set_vf_trust = nsim_set_vf_trust, 556 .ndo_get_vf_config = nsim_get_vf_config, 557 .ndo_set_vf_link_state = nsim_set_vf_link_state, 558 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 559 .ndo_setup_tc = nsim_setup_tc, 560 .ndo_set_features = nsim_set_features, 561 .ndo_get_iflink = nsim_get_iflink, 562 .ndo_bpf = nsim_bpf, 563 .ndo_open = nsim_open, 564 .ndo_stop = nsim_stop, 565 .net_shaper_ops = &nsim_shaper_ops, 566 }; 567 568 static const struct net_device_ops nsim_vf_netdev_ops = { 569 .ndo_start_xmit = nsim_start_xmit, 570 .ndo_set_rx_mode = nsim_set_rx_mode, 571 .ndo_set_mac_address = eth_mac_addr, 572 .ndo_validate_addr = eth_validate_addr, 573 .ndo_change_mtu = nsim_change_mtu, 574 .ndo_get_stats64 = nsim_get_stats64, 575 .ndo_setup_tc = nsim_setup_tc, 576 .ndo_set_features = nsim_set_features, 577 }; 578 579 /* We don't have true per-queue stats, yet, so do some random fakery here. 580 * Only report stuff for queue 0. 581 */ 582 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 583 struct netdev_queue_stats_rx *stats) 584 { 585 struct rtnl_link_stats64 rtstats = {}; 586 587 if (!idx) 588 nsim_get_stats64(dev, &rtstats); 589 590 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 591 stats->bytes = rtstats.rx_bytes; 592 } 593 594 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 595 struct netdev_queue_stats_tx *stats) 596 { 597 struct rtnl_link_stats64 rtstats = {}; 598 599 if (!idx) 600 nsim_get_stats64(dev, &rtstats); 601 602 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 603 stats->bytes = rtstats.tx_bytes; 604 } 605 606 static void nsim_get_base_stats(struct net_device *dev, 607 struct netdev_queue_stats_rx *rx, 608 struct netdev_queue_stats_tx *tx) 609 { 610 struct rtnl_link_stats64 rtstats = {}; 611 612 nsim_get_stats64(dev, &rtstats); 613 614 rx->packets = !!rtstats.rx_packets; 615 rx->bytes = 0; 616 tx->packets = !!rtstats.tx_packets; 617 tx->bytes = 0; 618 } 619 620 static const struct netdev_stat_ops nsim_stat_ops = { 621 .get_queue_stats_tx = nsim_get_queue_stats_tx, 622 .get_queue_stats_rx = nsim_get_queue_stats_rx, 623 .get_base_stats = nsim_get_base_stats, 624 }; 625 626 static struct nsim_rq *nsim_queue_alloc(void) 627 { 628 struct nsim_rq *rq; 629 630 rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT); 631 if (!rq) 632 return NULL; 633 634 skb_queue_head_init(&rq->skb_queue); 635 nsim_rq_timer_init(rq); 636 return rq; 637 } 638 639 static void nsim_queue_free(struct nsim_rq *rq) 640 { 641 hrtimer_cancel(&rq->napi_timer); 642 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 643 kfree(rq); 644 } 645 646 /* Queue reset mode is controlled by ns->rq_reset_mode. 647 * - normal - new NAPI new pool (old NAPI enabled when new added) 648 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 649 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 650 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 651 */ 652 struct nsim_queue_mem { 653 struct nsim_rq *rq; 654 struct page_pool *pp; 655 }; 656 657 static int 658 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx) 659 { 660 struct nsim_queue_mem *qmem = per_queue_mem; 661 struct netdevsim *ns = netdev_priv(dev); 662 int err; 663 664 if (ns->rq_reset_mode > 3) 665 return -EINVAL; 666 667 if (ns->rq_reset_mode == 1) { 668 if (!netif_running(ns->netdev)) 669 return -ENETDOWN; 670 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 671 } 672 673 qmem->rq = nsim_queue_alloc(); 674 if (!qmem->rq) 675 return -ENOMEM; 676 677 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 678 if (err) 679 goto err_free; 680 681 if (!ns->rq_reset_mode) 682 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 683 684 return 0; 685 686 err_free: 687 nsim_queue_free(qmem->rq); 688 return err; 689 } 690 691 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 692 { 693 struct nsim_queue_mem *qmem = per_queue_mem; 694 struct netdevsim *ns = netdev_priv(dev); 695 696 page_pool_destroy(qmem->pp); 697 if (qmem->rq) { 698 if (!ns->rq_reset_mode) 699 netif_napi_del(&qmem->rq->napi); 700 page_pool_destroy(qmem->rq->page_pool); 701 nsim_queue_free(qmem->rq); 702 } 703 } 704 705 static int 706 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) 707 { 708 struct nsim_queue_mem *qmem = per_queue_mem; 709 struct netdevsim *ns = netdev_priv(dev); 710 711 if (ns->rq_reset_mode == 1) { 712 ns->rq[idx]->page_pool = qmem->pp; 713 napi_enable(&ns->rq[idx]->napi); 714 return 0; 715 } 716 717 /* netif_napi_add()/_del() should normally be called from alloc/free, 718 * here we want to test various call orders. 719 */ 720 if (ns->rq_reset_mode == 2) { 721 netif_napi_del(&ns->rq[idx]->napi); 722 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 723 } else if (ns->rq_reset_mode == 3) { 724 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 725 netif_napi_del(&ns->rq[idx]->napi); 726 } 727 728 ns->rq[idx] = qmem->rq; 729 napi_enable(&ns->rq[idx]->napi); 730 731 return 0; 732 } 733 734 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 735 { 736 struct nsim_queue_mem *qmem = per_queue_mem; 737 struct netdevsim *ns = netdev_priv(dev); 738 739 napi_disable(&ns->rq[idx]->napi); 740 741 if (ns->rq_reset_mode == 1) { 742 qmem->pp = ns->rq[idx]->page_pool; 743 page_pool_disable_direct_recycling(qmem->pp); 744 } else { 745 qmem->rq = ns->rq[idx]; 746 } 747 748 return 0; 749 } 750 751 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 752 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 753 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 754 .ndo_queue_mem_free = nsim_queue_mem_free, 755 .ndo_queue_start = nsim_queue_start, 756 .ndo_queue_stop = nsim_queue_stop, 757 }; 758 759 static ssize_t 760 nsim_qreset_write(struct file *file, const char __user *data, 761 size_t count, loff_t *ppos) 762 { 763 struct netdevsim *ns = file->private_data; 764 unsigned int queue, mode; 765 char buf[32]; 766 ssize_t ret; 767 768 if (count >= sizeof(buf)) 769 return -EINVAL; 770 if (copy_from_user(buf, data, count)) 771 return -EFAULT; 772 buf[count] = '\0'; 773 774 ret = sscanf(buf, "%u %u", &queue, &mode); 775 if (ret != 2) 776 return -EINVAL; 777 778 rtnl_lock(); 779 if (queue >= ns->netdev->real_num_rx_queues) { 780 ret = -EINVAL; 781 goto exit_unlock; 782 } 783 784 ns->rq_reset_mode = mode; 785 ret = netdev_rx_queue_restart(ns->netdev, queue); 786 ns->rq_reset_mode = 0; 787 if (ret) 788 goto exit_unlock; 789 790 ret = count; 791 exit_unlock: 792 rtnl_unlock(); 793 return ret; 794 } 795 796 static const struct file_operations nsim_qreset_fops = { 797 .open = simple_open, 798 .write = nsim_qreset_write, 799 .owner = THIS_MODULE, 800 }; 801 802 static ssize_t 803 nsim_pp_hold_read(struct file *file, char __user *data, 804 size_t count, loff_t *ppos) 805 { 806 struct netdevsim *ns = file->private_data; 807 char buf[3] = "n\n"; 808 809 if (ns->page) 810 buf[0] = 'y'; 811 812 return simple_read_from_buffer(data, count, ppos, buf, 2); 813 } 814 815 static ssize_t 816 nsim_pp_hold_write(struct file *file, const char __user *data, 817 size_t count, loff_t *ppos) 818 { 819 struct netdevsim *ns = file->private_data; 820 ssize_t ret; 821 bool val; 822 823 ret = kstrtobool_from_user(data, count, &val); 824 if (ret) 825 return ret; 826 827 rtnl_lock(); 828 ret = count; 829 if (val == !!ns->page) 830 goto exit; 831 832 if (!netif_running(ns->netdev) && val) { 833 ret = -ENETDOWN; 834 } else if (val) { 835 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 836 if (!ns->page) 837 ret = -ENOMEM; 838 } else { 839 page_pool_put_full_page(ns->page->pp, ns->page, false); 840 ns->page = NULL; 841 } 842 843 exit: 844 rtnl_unlock(); 845 return ret; 846 } 847 848 static const struct file_operations nsim_pp_hold_fops = { 849 .open = simple_open, 850 .read = nsim_pp_hold_read, 851 .write = nsim_pp_hold_write, 852 .llseek = generic_file_llseek, 853 .owner = THIS_MODULE, 854 }; 855 856 static void nsim_setup(struct net_device *dev) 857 { 858 ether_setup(dev); 859 eth_hw_addr_random(dev); 860 861 dev->tx_queue_len = 0; 862 dev->flags &= ~IFF_MULTICAST; 863 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | 864 IFF_NO_QUEUE; 865 dev->features |= NETIF_F_HIGHDMA | 866 NETIF_F_SG | 867 NETIF_F_FRAGLIST | 868 NETIF_F_HW_CSUM | 869 NETIF_F_TSO; 870 dev->hw_features |= NETIF_F_HW_TC | 871 NETIF_F_SG | 872 NETIF_F_FRAGLIST | 873 NETIF_F_HW_CSUM | 874 NETIF_F_TSO; 875 dev->max_mtu = ETH_MAX_MTU; 876 dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD; 877 } 878 879 static int nsim_queue_init(struct netdevsim *ns) 880 { 881 struct net_device *dev = ns->netdev; 882 int i; 883 884 ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq), 885 GFP_KERNEL_ACCOUNT); 886 if (!ns->rq) 887 return -ENOMEM; 888 889 for (i = 0; i < dev->num_rx_queues; i++) { 890 ns->rq[i] = nsim_queue_alloc(); 891 if (!ns->rq[i]) 892 goto err_free_prev; 893 } 894 895 return 0; 896 897 err_free_prev: 898 while (i--) 899 kfree(ns->rq[i]); 900 kfree(ns->rq); 901 return -ENOMEM; 902 } 903 904 static void nsim_queue_uninit(struct netdevsim *ns) 905 { 906 struct net_device *dev = ns->netdev; 907 int i; 908 909 for (i = 0; i < dev->num_rx_queues; i++) 910 nsim_queue_free(ns->rq[i]); 911 912 kfree(ns->rq); 913 ns->rq = NULL; 914 } 915 916 static int nsim_init_netdevsim(struct netdevsim *ns) 917 { 918 struct mock_phc *phc; 919 int err; 920 921 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 922 if (IS_ERR(phc)) 923 return PTR_ERR(phc); 924 925 ns->phc = phc; 926 ns->netdev->netdev_ops = &nsim_netdev_ops; 927 ns->netdev->stat_ops = &nsim_stat_ops; 928 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 929 930 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 931 if (err) 932 goto err_phc_destroy; 933 934 rtnl_lock(); 935 err = nsim_queue_init(ns); 936 if (err) 937 goto err_utn_destroy; 938 939 err = nsim_bpf_init(ns); 940 if (err) 941 goto err_rq_destroy; 942 943 nsim_macsec_init(ns); 944 nsim_ipsec_init(ns); 945 946 err = register_netdevice(ns->netdev); 947 if (err) 948 goto err_ipsec_teardown; 949 rtnl_unlock(); 950 return 0; 951 952 err_ipsec_teardown: 953 nsim_ipsec_teardown(ns); 954 nsim_macsec_teardown(ns); 955 nsim_bpf_uninit(ns); 956 err_rq_destroy: 957 nsim_queue_uninit(ns); 958 err_utn_destroy: 959 rtnl_unlock(); 960 nsim_udp_tunnels_info_destroy(ns->netdev); 961 err_phc_destroy: 962 mock_phc_destroy(ns->phc); 963 return err; 964 } 965 966 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 967 { 968 int err; 969 970 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 971 rtnl_lock(); 972 err = register_netdevice(ns->netdev); 973 rtnl_unlock(); 974 return err; 975 } 976 977 static void nsim_exit_netdevsim(struct netdevsim *ns) 978 { 979 nsim_udp_tunnels_info_destroy(ns->netdev); 980 mock_phc_destroy(ns->phc); 981 } 982 983 struct netdevsim * 984 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port) 985 { 986 struct net_device *dev; 987 struct netdevsim *ns; 988 int err; 989 990 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 991 nsim_dev->nsim_bus_dev->num_queues); 992 if (!dev) 993 return ERR_PTR(-ENOMEM); 994 995 dev_net_set(dev, nsim_dev_net(nsim_dev)); 996 ns = netdev_priv(dev); 997 ns->netdev = dev; 998 u64_stats_init(&ns->syncp); 999 ns->nsim_dev = nsim_dev; 1000 ns->nsim_dev_port = nsim_dev_port; 1001 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1002 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1003 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1004 nsim_ethtool_init(ns); 1005 if (nsim_dev_port_is_pf(nsim_dev_port)) 1006 err = nsim_init_netdevsim(ns); 1007 else 1008 err = nsim_init_netdevsim_vf(ns); 1009 if (err) 1010 goto err_free_netdev; 1011 1012 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1013 ns, &nsim_pp_hold_fops); 1014 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1015 nsim_dev_port->ddir, ns, 1016 &nsim_qreset_fops); 1017 1018 return ns; 1019 1020 err_free_netdev: 1021 free_netdev(dev); 1022 return ERR_PTR(err); 1023 } 1024 1025 void nsim_destroy(struct netdevsim *ns) 1026 { 1027 struct net_device *dev = ns->netdev; 1028 struct netdevsim *peer; 1029 1030 debugfs_remove(ns->qr_dfs); 1031 debugfs_remove(ns->pp_dfs); 1032 1033 rtnl_lock(); 1034 peer = rtnl_dereference(ns->peer); 1035 if (peer) 1036 RCU_INIT_POINTER(peer->peer, NULL); 1037 RCU_INIT_POINTER(ns->peer, NULL); 1038 unregister_netdevice(dev); 1039 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1040 nsim_macsec_teardown(ns); 1041 nsim_ipsec_teardown(ns); 1042 nsim_bpf_uninit(ns); 1043 nsim_queue_uninit(ns); 1044 } 1045 rtnl_unlock(); 1046 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1047 nsim_exit_netdevsim(ns); 1048 1049 /* Put this intentionally late to exercise the orphaning path */ 1050 if (ns->page) { 1051 page_pool_put_full_page(ns->page->pp, ns->page, false); 1052 ns->page = NULL; 1053 } 1054 1055 free_netdev(dev); 1056 } 1057 1058 bool netdev_is_nsim(struct net_device *dev) 1059 { 1060 return dev->netdev_ops == &nsim_netdev_ops; 1061 } 1062 1063 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1064 struct netlink_ext_ack *extack) 1065 { 1066 NL_SET_ERR_MSG_MOD(extack, 1067 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1068 return -EOPNOTSUPP; 1069 } 1070 1071 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1072 .kind = DRV_NAME, 1073 .validate = nsim_validate, 1074 }; 1075 1076 static int __init nsim_module_init(void) 1077 { 1078 int err; 1079 1080 err = nsim_dev_init(); 1081 if (err) 1082 return err; 1083 1084 err = nsim_bus_init(); 1085 if (err) 1086 goto err_dev_exit; 1087 1088 err = rtnl_link_register(&nsim_link_ops); 1089 if (err) 1090 goto err_bus_exit; 1091 1092 return 0; 1093 1094 err_bus_exit: 1095 nsim_bus_exit(); 1096 err_dev_exit: 1097 nsim_dev_exit(); 1098 return err; 1099 } 1100 1101 static void __exit nsim_module_exit(void) 1102 { 1103 rtnl_link_unregister(&nsim_link_ops); 1104 nsim_bus_exit(); 1105 nsim_dev_exit(); 1106 } 1107 1108 module_init(nsim_module_init); 1109 module_exit(nsim_module_exit); 1110 MODULE_LICENSE("GPL"); 1111 MODULE_DESCRIPTION("Simulated networking device for testing"); 1112 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1113