1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/kernel.h> 19 #include <linux/module.h> 20 #include <linux/netdevice.h> 21 #include <linux/slab.h> 22 #include <net/netdev_queues.h> 23 #include <net/netdev_rx_queue.h> 24 #include <net/page_pool/helpers.h> 25 #include <net/netlink.h> 26 #include <net/net_shaper.h> 27 #include <net/pkt_cls.h> 28 #include <net/rtnetlink.h> 29 #include <net/udp_tunnel.h> 30 31 #include "netdevsim.h" 32 33 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 34 35 #define NSIM_RING_SIZE 256 36 37 static int nsim_napi_rx(struct nsim_rq *rq, struct sk_buff *skb) 38 { 39 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 40 dev_kfree_skb_any(skb); 41 return NET_RX_DROP; 42 } 43 44 skb_queue_tail(&rq->skb_queue, skb); 45 return NET_RX_SUCCESS; 46 } 47 48 static int nsim_forward_skb(struct net_device *dev, struct sk_buff *skb, 49 struct nsim_rq *rq) 50 { 51 return __dev_forward_skb(dev, skb) ?: nsim_napi_rx(rq, skb); 52 } 53 54 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 55 { 56 struct netdevsim *ns = netdev_priv(dev); 57 struct net_device *peer_dev; 58 unsigned int len = skb->len; 59 struct netdevsim *peer_ns; 60 struct nsim_rq *rq; 61 int rxq; 62 63 rcu_read_lock(); 64 if (!nsim_ipsec_tx(ns, skb)) 65 goto out_drop_free; 66 67 peer_ns = rcu_dereference(ns->peer); 68 if (!peer_ns) 69 goto out_drop_free; 70 71 peer_dev = peer_ns->netdev; 72 rxq = skb_get_queue_mapping(skb); 73 if (rxq >= peer_dev->num_rx_queues) 74 rxq = rxq % peer_dev->num_rx_queues; 75 rq = peer_ns->rq[rxq]; 76 77 skb_tx_timestamp(skb); 78 if (unlikely(nsim_forward_skb(peer_dev, skb, rq) == NET_RX_DROP)) 79 goto out_drop_cnt; 80 81 napi_schedule(&rq->napi); 82 83 rcu_read_unlock(); 84 u64_stats_update_begin(&ns->syncp); 85 ns->tx_packets++; 86 ns->tx_bytes += len; 87 u64_stats_update_end(&ns->syncp); 88 return NETDEV_TX_OK; 89 90 out_drop_free: 91 dev_kfree_skb(skb); 92 out_drop_cnt: 93 rcu_read_unlock(); 94 u64_stats_update_begin(&ns->syncp); 95 ns->tx_dropped++; 96 u64_stats_update_end(&ns->syncp); 97 return NETDEV_TX_OK; 98 } 99 100 static void nsim_set_rx_mode(struct net_device *dev) 101 { 102 } 103 104 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 105 { 106 struct netdevsim *ns = netdev_priv(dev); 107 108 if (ns->xdp.prog && new_mtu > NSIM_XDP_MAX_MTU) 109 return -EBUSY; 110 111 WRITE_ONCE(dev->mtu, new_mtu); 112 113 return 0; 114 } 115 116 static void 117 nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 118 { 119 struct netdevsim *ns = netdev_priv(dev); 120 unsigned int start; 121 122 do { 123 start = u64_stats_fetch_begin(&ns->syncp); 124 stats->tx_bytes = ns->tx_bytes; 125 stats->tx_packets = ns->tx_packets; 126 stats->tx_dropped = ns->tx_dropped; 127 } while (u64_stats_fetch_retry(&ns->syncp, start)); 128 } 129 130 static int 131 nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 132 { 133 return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv); 134 } 135 136 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 137 { 138 struct netdevsim *ns = netdev_priv(dev); 139 struct nsim_dev *nsim_dev = ns->nsim_dev; 140 141 /* Only refuse multicast addresses, zero address can mean unset/any. */ 142 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 143 return -EINVAL; 144 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 145 146 return 0; 147 } 148 149 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 150 u16 vlan, u8 qos, __be16 vlan_proto) 151 { 152 struct netdevsim *ns = netdev_priv(dev); 153 struct nsim_dev *nsim_dev = ns->nsim_dev; 154 155 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 156 return -EINVAL; 157 158 nsim_dev->vfconfigs[vf].vlan = vlan; 159 nsim_dev->vfconfigs[vf].qos = qos; 160 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 161 162 return 0; 163 } 164 165 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 166 { 167 struct netdevsim *ns = netdev_priv(dev); 168 struct nsim_dev *nsim_dev = ns->nsim_dev; 169 170 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 171 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 172 return -EOPNOTSUPP; 173 } 174 175 if (vf >= nsim_dev_get_vfs(nsim_dev)) 176 return -EINVAL; 177 178 nsim_dev->vfconfigs[vf].min_tx_rate = min; 179 nsim_dev->vfconfigs[vf].max_tx_rate = max; 180 181 return 0; 182 } 183 184 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 185 { 186 struct netdevsim *ns = netdev_priv(dev); 187 struct nsim_dev *nsim_dev = ns->nsim_dev; 188 189 if (vf >= nsim_dev_get_vfs(nsim_dev)) 190 return -EINVAL; 191 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 192 193 return 0; 194 } 195 196 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 197 { 198 struct netdevsim *ns = netdev_priv(dev); 199 struct nsim_dev *nsim_dev = ns->nsim_dev; 200 201 if (vf >= nsim_dev_get_vfs(nsim_dev)) 202 return -EINVAL; 203 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 204 205 return 0; 206 } 207 208 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 209 { 210 struct netdevsim *ns = netdev_priv(dev); 211 struct nsim_dev *nsim_dev = ns->nsim_dev; 212 213 if (vf >= nsim_dev_get_vfs(nsim_dev)) 214 return -EINVAL; 215 nsim_dev->vfconfigs[vf].trusted = val; 216 217 return 0; 218 } 219 220 static int 221 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 222 { 223 struct netdevsim *ns = netdev_priv(dev); 224 struct nsim_dev *nsim_dev = ns->nsim_dev; 225 226 if (vf >= nsim_dev_get_vfs(nsim_dev)) 227 return -EINVAL; 228 229 ivi->vf = vf; 230 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 231 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 232 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 233 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 234 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 235 ivi->qos = nsim_dev->vfconfigs[vf].qos; 236 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 237 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 238 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 239 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 240 241 return 0; 242 } 243 244 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 245 { 246 struct netdevsim *ns = netdev_priv(dev); 247 struct nsim_dev *nsim_dev = ns->nsim_dev; 248 249 if (vf >= nsim_dev_get_vfs(nsim_dev)) 250 return -EINVAL; 251 252 switch (state) { 253 case IFLA_VF_LINK_STATE_AUTO: 254 case IFLA_VF_LINK_STATE_ENABLE: 255 case IFLA_VF_LINK_STATE_DISABLE: 256 break; 257 default: 258 return -EINVAL; 259 } 260 261 nsim_dev->vfconfigs[vf].link_state = state; 262 263 return 0; 264 } 265 266 static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats) 267 { 268 stats->window_drops = 0; 269 stats->tx_overruns = 0; 270 } 271 272 static int nsim_setup_tc_taprio(struct net_device *dev, 273 struct tc_taprio_qopt_offload *offload) 274 { 275 int err = 0; 276 277 switch (offload->cmd) { 278 case TAPRIO_CMD_REPLACE: 279 case TAPRIO_CMD_DESTROY: 280 break; 281 case TAPRIO_CMD_STATS: 282 nsim_taprio_stats(&offload->stats); 283 break; 284 default: 285 err = -EOPNOTSUPP; 286 } 287 288 return err; 289 } 290 291 static LIST_HEAD(nsim_block_cb_list); 292 293 static int 294 nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) 295 { 296 struct netdevsim *ns = netdev_priv(dev); 297 298 switch (type) { 299 case TC_SETUP_QDISC_TAPRIO: 300 return nsim_setup_tc_taprio(dev, type_data); 301 case TC_SETUP_BLOCK: 302 return flow_block_cb_setup_simple(type_data, 303 &nsim_block_cb_list, 304 nsim_setup_tc_block_cb, 305 ns, ns, true); 306 default: 307 return -EOPNOTSUPP; 308 } 309 } 310 311 static int 312 nsim_set_features(struct net_device *dev, netdev_features_t features) 313 { 314 struct netdevsim *ns = netdev_priv(dev); 315 316 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 317 return nsim_bpf_disable_tc(ns); 318 319 return 0; 320 } 321 322 static int nsim_get_iflink(const struct net_device *dev) 323 { 324 struct netdevsim *nsim, *peer; 325 int iflink; 326 327 nsim = netdev_priv(dev); 328 329 rcu_read_lock(); 330 peer = rcu_dereference(nsim->peer); 331 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 332 READ_ONCE(dev->ifindex); 333 rcu_read_unlock(); 334 335 return iflink; 336 } 337 338 static int nsim_rcv(struct nsim_rq *rq, int budget) 339 { 340 struct sk_buff *skb; 341 int i; 342 343 for (i = 0; i < budget; i++) { 344 if (skb_queue_empty(&rq->skb_queue)) 345 break; 346 347 skb = skb_dequeue(&rq->skb_queue); 348 netif_receive_skb(skb); 349 } 350 351 return i; 352 } 353 354 static int nsim_poll(struct napi_struct *napi, int budget) 355 { 356 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 357 int done; 358 359 done = nsim_rcv(rq, budget); 360 napi_complete(napi); 361 362 return done; 363 } 364 365 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 366 { 367 struct page_pool_params params = { 368 .order = 0, 369 .pool_size = NSIM_RING_SIZE, 370 .nid = NUMA_NO_NODE, 371 .dev = &napi->dev->dev, 372 .napi = napi, 373 .dma_dir = DMA_BIDIRECTIONAL, 374 .netdev = napi->dev, 375 }; 376 struct page_pool *pool; 377 378 pool = page_pool_create(¶ms); 379 if (IS_ERR(pool)) 380 return PTR_ERR(pool); 381 382 *p = pool; 383 return 0; 384 } 385 386 static int nsim_init_napi(struct netdevsim *ns) 387 { 388 struct net_device *dev = ns->netdev; 389 struct nsim_rq *rq; 390 int err, i; 391 392 for (i = 0; i < dev->num_rx_queues; i++) { 393 rq = ns->rq[i]; 394 395 netif_napi_add_config(dev, &rq->napi, nsim_poll, i); 396 } 397 398 for (i = 0; i < dev->num_rx_queues; i++) { 399 rq = ns->rq[i]; 400 401 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 402 if (err) 403 goto err_pp_destroy; 404 } 405 406 return 0; 407 408 err_pp_destroy: 409 while (i--) { 410 page_pool_destroy(ns->rq[i]->page_pool); 411 ns->rq[i]->page_pool = NULL; 412 } 413 414 for (i = 0; i < dev->num_rx_queues; i++) 415 __netif_napi_del(&ns->rq[i]->napi); 416 417 return err; 418 } 419 420 static void nsim_enable_napi(struct netdevsim *ns) 421 { 422 struct net_device *dev = ns->netdev; 423 int i; 424 425 for (i = 0; i < dev->num_rx_queues; i++) { 426 struct nsim_rq *rq = ns->rq[i]; 427 428 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 429 napi_enable(&rq->napi); 430 } 431 } 432 433 static int nsim_open(struct net_device *dev) 434 { 435 struct netdevsim *ns = netdev_priv(dev); 436 int err; 437 438 err = nsim_init_napi(ns); 439 if (err) 440 return err; 441 442 nsim_enable_napi(ns); 443 444 return 0; 445 } 446 447 static void nsim_del_napi(struct netdevsim *ns) 448 { 449 struct net_device *dev = ns->netdev; 450 int i; 451 452 for (i = 0; i < dev->num_rx_queues; i++) { 453 struct nsim_rq *rq = ns->rq[i]; 454 455 napi_disable(&rq->napi); 456 __netif_napi_del(&rq->napi); 457 } 458 synchronize_net(); 459 460 for (i = 0; i < dev->num_rx_queues; i++) { 461 page_pool_destroy(ns->rq[i]->page_pool); 462 ns->rq[i]->page_pool = NULL; 463 } 464 } 465 466 static int nsim_stop(struct net_device *dev) 467 { 468 struct netdevsim *ns = netdev_priv(dev); 469 struct netdevsim *peer; 470 471 netif_carrier_off(dev); 472 peer = rtnl_dereference(ns->peer); 473 if (peer) 474 netif_carrier_off(peer->netdev); 475 476 nsim_del_napi(ns); 477 478 return 0; 479 } 480 481 static int nsim_shaper_set(struct net_shaper_binding *binding, 482 const struct net_shaper *shaper, 483 struct netlink_ext_ack *extack) 484 { 485 return 0; 486 } 487 488 static int nsim_shaper_del(struct net_shaper_binding *binding, 489 const struct net_shaper_handle *handle, 490 struct netlink_ext_ack *extack) 491 { 492 return 0; 493 } 494 495 static int nsim_shaper_group(struct net_shaper_binding *binding, 496 int leaves_count, 497 const struct net_shaper *leaves, 498 const struct net_shaper *root, 499 struct netlink_ext_ack *extack) 500 { 501 return 0; 502 } 503 504 static void nsim_shaper_cap(struct net_shaper_binding *binding, 505 enum net_shaper_scope scope, 506 unsigned long *flags) 507 { 508 *flags = ULONG_MAX; 509 } 510 511 static const struct net_shaper_ops nsim_shaper_ops = { 512 .set = nsim_shaper_set, 513 .delete = nsim_shaper_del, 514 .group = nsim_shaper_group, 515 .capabilities = nsim_shaper_cap, 516 }; 517 518 static const struct net_device_ops nsim_netdev_ops = { 519 .ndo_start_xmit = nsim_start_xmit, 520 .ndo_set_rx_mode = nsim_set_rx_mode, 521 .ndo_set_mac_address = eth_mac_addr, 522 .ndo_validate_addr = eth_validate_addr, 523 .ndo_change_mtu = nsim_change_mtu, 524 .ndo_get_stats64 = nsim_get_stats64, 525 .ndo_set_vf_mac = nsim_set_vf_mac, 526 .ndo_set_vf_vlan = nsim_set_vf_vlan, 527 .ndo_set_vf_rate = nsim_set_vf_rate, 528 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 529 .ndo_set_vf_trust = nsim_set_vf_trust, 530 .ndo_get_vf_config = nsim_get_vf_config, 531 .ndo_set_vf_link_state = nsim_set_vf_link_state, 532 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 533 .ndo_setup_tc = nsim_setup_tc, 534 .ndo_set_features = nsim_set_features, 535 .ndo_get_iflink = nsim_get_iflink, 536 .ndo_bpf = nsim_bpf, 537 .ndo_open = nsim_open, 538 .ndo_stop = nsim_stop, 539 .net_shaper_ops = &nsim_shaper_ops, 540 }; 541 542 static const struct net_device_ops nsim_vf_netdev_ops = { 543 .ndo_start_xmit = nsim_start_xmit, 544 .ndo_set_rx_mode = nsim_set_rx_mode, 545 .ndo_set_mac_address = eth_mac_addr, 546 .ndo_validate_addr = eth_validate_addr, 547 .ndo_change_mtu = nsim_change_mtu, 548 .ndo_get_stats64 = nsim_get_stats64, 549 .ndo_setup_tc = nsim_setup_tc, 550 .ndo_set_features = nsim_set_features, 551 }; 552 553 /* We don't have true per-queue stats, yet, so do some random fakery here. 554 * Only report stuff for queue 0. 555 */ 556 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 557 struct netdev_queue_stats_rx *stats) 558 { 559 struct rtnl_link_stats64 rtstats = {}; 560 561 if (!idx) 562 nsim_get_stats64(dev, &rtstats); 563 564 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 565 stats->bytes = rtstats.rx_bytes; 566 } 567 568 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 569 struct netdev_queue_stats_tx *stats) 570 { 571 struct rtnl_link_stats64 rtstats = {}; 572 573 if (!idx) 574 nsim_get_stats64(dev, &rtstats); 575 576 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 577 stats->bytes = rtstats.tx_bytes; 578 } 579 580 static void nsim_get_base_stats(struct net_device *dev, 581 struct netdev_queue_stats_rx *rx, 582 struct netdev_queue_stats_tx *tx) 583 { 584 struct rtnl_link_stats64 rtstats = {}; 585 586 nsim_get_stats64(dev, &rtstats); 587 588 rx->packets = !!rtstats.rx_packets; 589 rx->bytes = 0; 590 tx->packets = !!rtstats.tx_packets; 591 tx->bytes = 0; 592 } 593 594 static const struct netdev_stat_ops nsim_stat_ops = { 595 .get_queue_stats_tx = nsim_get_queue_stats_tx, 596 .get_queue_stats_rx = nsim_get_queue_stats_rx, 597 .get_base_stats = nsim_get_base_stats, 598 }; 599 600 static struct nsim_rq *nsim_queue_alloc(void) 601 { 602 struct nsim_rq *rq; 603 604 rq = kzalloc(sizeof(*rq), GFP_KERNEL_ACCOUNT); 605 if (!rq) 606 return NULL; 607 608 skb_queue_head_init(&rq->skb_queue); 609 return rq; 610 } 611 612 static void nsim_queue_free(struct nsim_rq *rq) 613 { 614 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 615 kfree(rq); 616 } 617 618 /* Queue reset mode is controlled by ns->rq_reset_mode. 619 * - normal - new NAPI new pool (old NAPI enabled when new added) 620 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 621 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 622 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 623 */ 624 struct nsim_queue_mem { 625 struct nsim_rq *rq; 626 struct page_pool *pp; 627 }; 628 629 static int 630 nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx) 631 { 632 struct nsim_queue_mem *qmem = per_queue_mem; 633 struct netdevsim *ns = netdev_priv(dev); 634 int err; 635 636 if (ns->rq_reset_mode > 3) 637 return -EINVAL; 638 639 if (ns->rq_reset_mode == 1) 640 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 641 642 qmem->rq = nsim_queue_alloc(); 643 if (!qmem->rq) 644 return -ENOMEM; 645 646 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 647 if (err) 648 goto err_free; 649 650 if (!ns->rq_reset_mode) 651 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 652 653 return 0; 654 655 err_free: 656 nsim_queue_free(qmem->rq); 657 return err; 658 } 659 660 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 661 { 662 struct nsim_queue_mem *qmem = per_queue_mem; 663 struct netdevsim *ns = netdev_priv(dev); 664 665 page_pool_destroy(qmem->pp); 666 if (qmem->rq) { 667 if (!ns->rq_reset_mode) 668 netif_napi_del(&qmem->rq->napi); 669 page_pool_destroy(qmem->rq->page_pool); 670 nsim_queue_free(qmem->rq); 671 } 672 } 673 674 static int 675 nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) 676 { 677 struct nsim_queue_mem *qmem = per_queue_mem; 678 struct netdevsim *ns = netdev_priv(dev); 679 680 if (ns->rq_reset_mode == 1) { 681 ns->rq[idx]->page_pool = qmem->pp; 682 napi_enable(&ns->rq[idx]->napi); 683 return 0; 684 } 685 686 /* netif_napi_add()/_del() should normally be called from alloc/free, 687 * here we want to test various call orders. 688 */ 689 if (ns->rq_reset_mode == 2) { 690 netif_napi_del(&ns->rq[idx]->napi); 691 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 692 } else if (ns->rq_reset_mode == 3) { 693 netif_napi_add_config(dev, &qmem->rq->napi, nsim_poll, idx); 694 netif_napi_del(&ns->rq[idx]->napi); 695 } 696 697 ns->rq[idx] = qmem->rq; 698 napi_enable(&ns->rq[idx]->napi); 699 700 return 0; 701 } 702 703 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 704 { 705 struct nsim_queue_mem *qmem = per_queue_mem; 706 struct netdevsim *ns = netdev_priv(dev); 707 708 napi_disable(&ns->rq[idx]->napi); 709 710 if (ns->rq_reset_mode == 1) { 711 qmem->pp = ns->rq[idx]->page_pool; 712 page_pool_disable_direct_recycling(qmem->pp); 713 } else { 714 qmem->rq = ns->rq[idx]; 715 } 716 717 return 0; 718 } 719 720 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 721 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 722 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 723 .ndo_queue_mem_free = nsim_queue_mem_free, 724 .ndo_queue_start = nsim_queue_start, 725 .ndo_queue_stop = nsim_queue_stop, 726 }; 727 728 static ssize_t 729 nsim_qreset_write(struct file *file, const char __user *data, 730 size_t count, loff_t *ppos) 731 { 732 struct netdevsim *ns = file->private_data; 733 unsigned int queue, mode; 734 char buf[32]; 735 ssize_t ret; 736 737 if (count >= sizeof(buf)) 738 return -EINVAL; 739 if (copy_from_user(buf, data, count)) 740 return -EFAULT; 741 buf[count] = '\0'; 742 743 ret = sscanf(buf, "%u %u", &queue, &mode); 744 if (ret != 2) 745 return -EINVAL; 746 747 rtnl_lock(); 748 if (!netif_running(ns->netdev)) { 749 ret = -ENETDOWN; 750 goto exit_unlock; 751 } 752 753 if (queue >= ns->netdev->real_num_rx_queues) { 754 ret = -EINVAL; 755 goto exit_unlock; 756 } 757 758 ns->rq_reset_mode = mode; 759 ret = netdev_rx_queue_restart(ns->netdev, queue); 760 ns->rq_reset_mode = 0; 761 if (ret) 762 goto exit_unlock; 763 764 ret = count; 765 exit_unlock: 766 rtnl_unlock(); 767 return ret; 768 } 769 770 static const struct file_operations nsim_qreset_fops = { 771 .open = simple_open, 772 .write = nsim_qreset_write, 773 .owner = THIS_MODULE, 774 }; 775 776 static ssize_t 777 nsim_pp_hold_read(struct file *file, char __user *data, 778 size_t count, loff_t *ppos) 779 { 780 struct netdevsim *ns = file->private_data; 781 char buf[3] = "n\n"; 782 783 if (ns->page) 784 buf[0] = 'y'; 785 786 return simple_read_from_buffer(data, count, ppos, buf, 2); 787 } 788 789 static ssize_t 790 nsim_pp_hold_write(struct file *file, const char __user *data, 791 size_t count, loff_t *ppos) 792 { 793 struct netdevsim *ns = file->private_data; 794 ssize_t ret; 795 bool val; 796 797 ret = kstrtobool_from_user(data, count, &val); 798 if (ret) 799 return ret; 800 801 rtnl_lock(); 802 ret = count; 803 if (val == !!ns->page) 804 goto exit; 805 806 if (!netif_running(ns->netdev) && val) { 807 ret = -ENETDOWN; 808 } else if (val) { 809 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 810 if (!ns->page) 811 ret = -ENOMEM; 812 } else { 813 page_pool_put_full_page(ns->page->pp, ns->page, false); 814 ns->page = NULL; 815 } 816 817 exit: 818 rtnl_unlock(); 819 return ret; 820 } 821 822 static const struct file_operations nsim_pp_hold_fops = { 823 .open = simple_open, 824 .read = nsim_pp_hold_read, 825 .write = nsim_pp_hold_write, 826 .llseek = generic_file_llseek, 827 .owner = THIS_MODULE, 828 }; 829 830 static void nsim_setup(struct net_device *dev) 831 { 832 ether_setup(dev); 833 eth_hw_addr_random(dev); 834 835 dev->tx_queue_len = 0; 836 dev->flags &= ~IFF_MULTICAST; 837 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | 838 IFF_NO_QUEUE; 839 dev->features |= NETIF_F_HIGHDMA | 840 NETIF_F_SG | 841 NETIF_F_FRAGLIST | 842 NETIF_F_HW_CSUM | 843 NETIF_F_TSO; 844 dev->hw_features |= NETIF_F_HW_TC | 845 NETIF_F_SG | 846 NETIF_F_FRAGLIST | 847 NETIF_F_HW_CSUM | 848 NETIF_F_TSO; 849 dev->max_mtu = ETH_MAX_MTU; 850 dev->xdp_features = NETDEV_XDP_ACT_HW_OFFLOAD; 851 } 852 853 static int nsim_queue_init(struct netdevsim *ns) 854 { 855 struct net_device *dev = ns->netdev; 856 int i; 857 858 ns->rq = kcalloc(dev->num_rx_queues, sizeof(*ns->rq), 859 GFP_KERNEL_ACCOUNT); 860 if (!ns->rq) 861 return -ENOMEM; 862 863 for (i = 0; i < dev->num_rx_queues; i++) { 864 ns->rq[i] = nsim_queue_alloc(); 865 if (!ns->rq[i]) 866 goto err_free_prev; 867 } 868 869 return 0; 870 871 err_free_prev: 872 while (i--) 873 kfree(ns->rq[i]); 874 kfree(ns->rq); 875 return -ENOMEM; 876 } 877 878 static void nsim_queue_uninit(struct netdevsim *ns) 879 { 880 struct net_device *dev = ns->netdev; 881 int i; 882 883 for (i = 0; i < dev->num_rx_queues; i++) 884 nsim_queue_free(ns->rq[i]); 885 886 kfree(ns->rq); 887 ns->rq = NULL; 888 } 889 890 static int nsim_init_netdevsim(struct netdevsim *ns) 891 { 892 struct mock_phc *phc; 893 int err; 894 895 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 896 if (IS_ERR(phc)) 897 return PTR_ERR(phc); 898 899 ns->phc = phc; 900 ns->netdev->netdev_ops = &nsim_netdev_ops; 901 ns->netdev->stat_ops = &nsim_stat_ops; 902 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 903 904 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 905 if (err) 906 goto err_phc_destroy; 907 908 rtnl_lock(); 909 err = nsim_queue_init(ns); 910 if (err) 911 goto err_utn_destroy; 912 913 err = nsim_bpf_init(ns); 914 if (err) 915 goto err_rq_destroy; 916 917 nsim_macsec_init(ns); 918 nsim_ipsec_init(ns); 919 920 err = register_netdevice(ns->netdev); 921 if (err) 922 goto err_ipsec_teardown; 923 rtnl_unlock(); 924 return 0; 925 926 err_ipsec_teardown: 927 nsim_ipsec_teardown(ns); 928 nsim_macsec_teardown(ns); 929 nsim_bpf_uninit(ns); 930 err_rq_destroy: 931 nsim_queue_uninit(ns); 932 err_utn_destroy: 933 rtnl_unlock(); 934 nsim_udp_tunnels_info_destroy(ns->netdev); 935 err_phc_destroy: 936 mock_phc_destroy(ns->phc); 937 return err; 938 } 939 940 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 941 { 942 int err; 943 944 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 945 rtnl_lock(); 946 err = register_netdevice(ns->netdev); 947 rtnl_unlock(); 948 return err; 949 } 950 951 static void nsim_exit_netdevsim(struct netdevsim *ns) 952 { 953 nsim_udp_tunnels_info_destroy(ns->netdev); 954 mock_phc_destroy(ns->phc); 955 } 956 957 struct netdevsim * 958 nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port) 959 { 960 struct net_device *dev; 961 struct netdevsim *ns; 962 int err; 963 964 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 965 nsim_dev->nsim_bus_dev->num_queues); 966 if (!dev) 967 return ERR_PTR(-ENOMEM); 968 969 dev_net_set(dev, nsim_dev_net(nsim_dev)); 970 ns = netdev_priv(dev); 971 ns->netdev = dev; 972 u64_stats_init(&ns->syncp); 973 ns->nsim_dev = nsim_dev; 974 ns->nsim_dev_port = nsim_dev_port; 975 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 976 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 977 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 978 nsim_ethtool_init(ns); 979 if (nsim_dev_port_is_pf(nsim_dev_port)) 980 err = nsim_init_netdevsim(ns); 981 else 982 err = nsim_init_netdevsim_vf(ns); 983 if (err) 984 goto err_free_netdev; 985 986 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 987 ns, &nsim_pp_hold_fops); 988 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 989 nsim_dev_port->ddir, ns, 990 &nsim_qreset_fops); 991 992 return ns; 993 994 err_free_netdev: 995 free_netdev(dev); 996 return ERR_PTR(err); 997 } 998 999 void nsim_destroy(struct netdevsim *ns) 1000 { 1001 struct net_device *dev = ns->netdev; 1002 struct netdevsim *peer; 1003 1004 debugfs_remove(ns->qr_dfs); 1005 debugfs_remove(ns->pp_dfs); 1006 1007 rtnl_lock(); 1008 peer = rtnl_dereference(ns->peer); 1009 if (peer) 1010 RCU_INIT_POINTER(peer->peer, NULL); 1011 RCU_INIT_POINTER(ns->peer, NULL); 1012 unregister_netdevice(dev); 1013 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1014 nsim_macsec_teardown(ns); 1015 nsim_ipsec_teardown(ns); 1016 nsim_bpf_uninit(ns); 1017 nsim_queue_uninit(ns); 1018 } 1019 rtnl_unlock(); 1020 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1021 nsim_exit_netdevsim(ns); 1022 1023 /* Put this intentionally late to exercise the orphaning path */ 1024 if (ns->page) { 1025 page_pool_put_full_page(ns->page->pp, ns->page, false); 1026 ns->page = NULL; 1027 } 1028 1029 free_netdev(dev); 1030 } 1031 1032 bool netdev_is_nsim(struct net_device *dev) 1033 { 1034 return dev->netdev_ops == &nsim_netdev_ops; 1035 } 1036 1037 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1038 struct netlink_ext_ack *extack) 1039 { 1040 NL_SET_ERR_MSG_MOD(extack, 1041 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1042 return -EOPNOTSUPP; 1043 } 1044 1045 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1046 .kind = DRV_NAME, 1047 .validate = nsim_validate, 1048 }; 1049 1050 static int __init nsim_module_init(void) 1051 { 1052 int err; 1053 1054 err = nsim_dev_init(); 1055 if (err) 1056 return err; 1057 1058 err = nsim_bus_init(); 1059 if (err) 1060 goto err_dev_exit; 1061 1062 err = rtnl_link_register(&nsim_link_ops); 1063 if (err) 1064 goto err_bus_exit; 1065 1066 return 0; 1067 1068 err_bus_exit: 1069 nsim_bus_exit(); 1070 err_dev_exit: 1071 nsim_dev_exit(); 1072 return err; 1073 } 1074 1075 static void __exit nsim_module_exit(void) 1076 { 1077 rtnl_link_unregister(&nsim_link_ops); 1078 nsim_bus_exit(); 1079 nsim_dev_exit(); 1080 } 1081 1082 module_init(nsim_module_init); 1083 module_exit(nsim_module_exit); 1084 MODULE_LICENSE("GPL"); 1085 MODULE_DESCRIPTION("Simulated networking device for testing"); 1086 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1087