1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 #include <net/busy_poll.h> 33 34 #include "netdevsim.h" 35 36 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 37 38 #define NSIM_RING_SIZE 256 39 40 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq) 41 { 42 struct netdevsim *ns = netdev_priv(dev); 43 struct net_device *peer_dev; 44 struct netdevsim *peer_ns; 45 struct netdev_queue *txq; 46 u16 idx; 47 48 idx = rq->napi.index; 49 rcu_read_lock(); 50 peer_ns = rcu_dereference(ns->peer); 51 if (!peer_ns) 52 goto out; 53 54 /* TX device */ 55 peer_dev = peer_ns->netdev; 56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues) 57 goto out; 58 59 txq = netdev_get_tx_queue(peer_dev, idx); 60 if (!netif_tx_queue_stopped(txq)) 61 goto out; 62 63 netif_tx_wake_queue(txq); 64 out: 65 rcu_read_unlock(); 66 } 67 68 static void nsim_stop_tx_queue(struct net_device *tx_dev, 69 struct net_device *rx_dev, 70 struct nsim_rq *rq, 71 u16 idx) 72 { 73 /* If different queues size, do not stop, since it is not 74 * easy to find which TX queue is mapped here 75 */ 76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) 77 return; 78 79 /* rq is the queue on the receive side */ 80 netif_subqueue_try_stop(tx_dev, idx, 81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), 82 NSIM_RING_SIZE / 2); 83 } 84 85 static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, 86 struct nsim_rq *rq, struct sk_buff *skb) 87 { 88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 89 dev_kfree_skb_any(skb); 90 return NET_RX_DROP; 91 } 92 93 skb_queue_tail(&rq->skb_queue, skb); 94 95 /* Stop the peer TX queue avoiding dropping packets later */ 96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) 97 nsim_stop_tx_queue(tx_dev, rx_dev, rq, 98 skb_get_queue_mapping(skb)); 99 100 return NET_RX_SUCCESS; 101 } 102 103 static int nsim_forward_skb(struct net_device *tx_dev, 104 struct net_device *rx_dev, 105 struct sk_buff *skb, 106 struct nsim_rq *rq, 107 struct skb_ext *psp_ext) 108 { 109 int ret; 110 111 ret = __dev_forward_skb(rx_dev, skb); 112 if (ret) { 113 if (psp_ext) 114 __skb_ext_put(psp_ext); 115 return ret; 116 } 117 118 nsim_psp_handle_ext(skb, psp_ext); 119 120 return nsim_napi_rx(tx_dev, rx_dev, rq, skb); 121 } 122 123 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 124 { 125 struct netdevsim *ns = netdev_priv(dev); 126 struct skb_ext *psp_ext = NULL; 127 struct net_device *peer_dev; 128 unsigned int len = skb->len; 129 struct netdevsim *peer_ns; 130 struct netdev_config *cfg; 131 struct nsim_rq *rq; 132 int rxq; 133 int dr; 134 135 rcu_read_lock(); 136 if (!nsim_ipsec_tx(ns, skb)) 137 goto out_drop_any; 138 139 /* Check if loopback mode is enabled */ 140 if (dev->features & NETIF_F_LOOPBACK) { 141 peer_ns = ns; 142 peer_dev = dev; 143 } else { 144 peer_ns = rcu_dereference(ns->peer); 145 if (!peer_ns) 146 goto out_drop_any; 147 peer_dev = peer_ns->netdev; 148 } 149 150 dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext); 151 if (dr) 152 goto out_drop_free; 153 154 rxq = skb_get_queue_mapping(skb); 155 if (rxq >= peer_dev->num_rx_queues) 156 rxq = rxq % peer_dev->num_rx_queues; 157 rq = peer_ns->rq[rxq]; 158 159 cfg = peer_dev->cfg; 160 if (skb_is_nonlinear(skb) && 161 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 162 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 163 cfg->hds_thresh > len))) 164 skb_linearize(skb); 165 166 skb_tx_timestamp(skb); 167 if (unlikely(nsim_forward_skb(dev, peer_dev, 168 skb, rq, psp_ext) == NET_RX_DROP)) 169 goto out_drop_cnt; 170 171 if (!hrtimer_active(&rq->napi_timer)) 172 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 173 174 rcu_read_unlock(); 175 dev_dstats_tx_add(dev, len); 176 return NETDEV_TX_OK; 177 178 out_drop_any: 179 dr = SKB_DROP_REASON_NOT_SPECIFIED; 180 out_drop_free: 181 kfree_skb_reason(skb, dr); 182 out_drop_cnt: 183 rcu_read_unlock(); 184 dev_dstats_tx_dropped(dev); 185 return NETDEV_TX_OK; 186 } 187 188 static void nsim_set_rx_mode(struct net_device *dev) 189 { 190 } 191 192 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 193 { 194 struct netdevsim *ns = netdev_priv(dev); 195 196 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 197 new_mtu > NSIM_XDP_MAX_MTU) 198 return -EBUSY; 199 200 WRITE_ONCE(dev->mtu, new_mtu); 201 202 return 0; 203 } 204 205 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 206 { 207 struct netdevsim *ns = netdev_priv(dev); 208 struct nsim_dev *nsim_dev = ns->nsim_dev; 209 210 /* Only refuse multicast addresses, zero address can mean unset/any. */ 211 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 212 return -EINVAL; 213 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 214 215 return 0; 216 } 217 218 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 219 u16 vlan, u8 qos, __be16 vlan_proto) 220 { 221 struct netdevsim *ns = netdev_priv(dev); 222 struct nsim_dev *nsim_dev = ns->nsim_dev; 223 224 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 225 return -EINVAL; 226 227 nsim_dev->vfconfigs[vf].vlan = vlan; 228 nsim_dev->vfconfigs[vf].qos = qos; 229 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 230 231 return 0; 232 } 233 234 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 235 { 236 struct netdevsim *ns = netdev_priv(dev); 237 struct nsim_dev *nsim_dev = ns->nsim_dev; 238 239 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 240 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 241 return -EOPNOTSUPP; 242 } 243 244 if (vf >= nsim_dev_get_vfs(nsim_dev)) 245 return -EINVAL; 246 247 nsim_dev->vfconfigs[vf].min_tx_rate = min; 248 nsim_dev->vfconfigs[vf].max_tx_rate = max; 249 250 return 0; 251 } 252 253 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 254 { 255 struct netdevsim *ns = netdev_priv(dev); 256 struct nsim_dev *nsim_dev = ns->nsim_dev; 257 258 if (vf >= nsim_dev_get_vfs(nsim_dev)) 259 return -EINVAL; 260 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 261 262 return 0; 263 } 264 265 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 266 { 267 struct netdevsim *ns = netdev_priv(dev); 268 struct nsim_dev *nsim_dev = ns->nsim_dev; 269 270 if (vf >= nsim_dev_get_vfs(nsim_dev)) 271 return -EINVAL; 272 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 273 274 return 0; 275 } 276 277 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 278 { 279 struct netdevsim *ns = netdev_priv(dev); 280 struct nsim_dev *nsim_dev = ns->nsim_dev; 281 282 if (vf >= nsim_dev_get_vfs(nsim_dev)) 283 return -EINVAL; 284 nsim_dev->vfconfigs[vf].trusted = val; 285 286 return 0; 287 } 288 289 static int 290 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 291 { 292 struct netdevsim *ns = netdev_priv(dev); 293 struct nsim_dev *nsim_dev = ns->nsim_dev; 294 295 if (vf >= nsim_dev_get_vfs(nsim_dev)) 296 return -EINVAL; 297 298 ivi->vf = vf; 299 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 300 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 301 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 302 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 303 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 304 ivi->qos = nsim_dev->vfconfigs[vf].qos; 305 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 306 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 307 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 308 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 309 310 return 0; 311 } 312 313 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 314 { 315 struct netdevsim *ns = netdev_priv(dev); 316 struct nsim_dev *nsim_dev = ns->nsim_dev; 317 318 if (vf >= nsim_dev_get_vfs(nsim_dev)) 319 return -EINVAL; 320 321 switch (state) { 322 case IFLA_VF_LINK_STATE_AUTO: 323 case IFLA_VF_LINK_STATE_ENABLE: 324 case IFLA_VF_LINK_STATE_DISABLE: 325 break; 326 default: 327 return -EINVAL; 328 } 329 330 nsim_dev->vfconfigs[vf].link_state = state; 331 332 return 0; 333 } 334 335 static int 336 nsim_set_features(struct net_device *dev, netdev_features_t features) 337 { 338 struct netdevsim *ns = netdev_priv(dev); 339 340 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 341 return nsim_bpf_disable_tc(ns); 342 343 return 0; 344 } 345 346 static int nsim_get_iflink(const struct net_device *dev) 347 { 348 struct netdevsim *nsim, *peer; 349 int iflink; 350 351 nsim = netdev_priv(dev); 352 353 rcu_read_lock(); 354 peer = rcu_dereference(nsim->peer); 355 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 356 READ_ONCE(dev->ifindex); 357 rcu_read_unlock(); 358 359 return iflink; 360 } 361 362 static int nsim_rcv(struct nsim_rq *rq, int budget) 363 { 364 struct net_device *dev = rq->napi.dev; 365 struct bpf_prog *xdp_prog; 366 struct netdevsim *ns; 367 struct sk_buff *skb; 368 unsigned int skblen; 369 int i, ret; 370 371 ns = netdev_priv(dev); 372 xdp_prog = READ_ONCE(ns->xdp.prog); 373 374 for (i = 0; i < budget; i++) { 375 if (skb_queue_empty(&rq->skb_queue)) 376 break; 377 378 skb = skb_dequeue(&rq->skb_queue); 379 380 if (xdp_prog) { 381 /* skb might be freed directly by XDP, save the len */ 382 skblen = skb->len; 383 384 if (skb->ip_summed == CHECKSUM_PARTIAL) 385 skb_checksum_help(skb); 386 ret = do_xdp_generic(xdp_prog, &skb); 387 if (ret != XDP_PASS) { 388 dev_dstats_rx_add(dev, skblen); 389 continue; 390 } 391 } 392 393 /* skb might be discard at netif_receive_skb, save the len */ 394 dev_dstats_rx_add(dev, skb->len); 395 napi_gro_receive(&rq->napi, skb); 396 } 397 398 nsim_start_peer_tx_queue(dev, rq); 399 return i; 400 } 401 402 static int nsim_poll(struct napi_struct *napi, int budget) 403 { 404 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 405 int done; 406 407 done = nsim_rcv(rq, budget); 408 if (done < budget) 409 napi_complete_done(napi, done); 410 411 return done; 412 } 413 414 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 415 { 416 struct page_pool_params params = { 417 .order = 0, 418 .pool_size = NSIM_RING_SIZE, 419 .nid = NUMA_NO_NODE, 420 .dev = &napi->dev->dev, 421 .napi = napi, 422 .dma_dir = DMA_BIDIRECTIONAL, 423 .netdev = napi->dev, 424 }; 425 struct page_pool *pool; 426 427 pool = page_pool_create(¶ms); 428 if (IS_ERR(pool)) 429 return PTR_ERR(pool); 430 431 *p = pool; 432 return 0; 433 } 434 435 static int nsim_init_napi(struct netdevsim *ns) 436 { 437 struct net_device *dev = ns->netdev; 438 struct nsim_rq *rq; 439 int err, i; 440 441 for (i = 0; i < dev->num_rx_queues; i++) { 442 rq = ns->rq[i]; 443 444 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 445 } 446 447 for (i = 0; i < dev->num_rx_queues; i++) { 448 rq = ns->rq[i]; 449 450 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 451 if (err) 452 goto err_pp_destroy; 453 } 454 455 return 0; 456 457 err_pp_destroy: 458 while (i--) { 459 page_pool_destroy(ns->rq[i]->page_pool); 460 ns->rq[i]->page_pool = NULL; 461 } 462 463 for (i = 0; i < dev->num_rx_queues; i++) 464 __netif_napi_del_locked(&ns->rq[i]->napi); 465 466 return err; 467 } 468 469 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 470 { 471 struct nsim_rq *rq; 472 473 rq = container_of(timer, struct nsim_rq, napi_timer); 474 napi_schedule(&rq->napi); 475 476 return HRTIMER_NORESTART; 477 } 478 479 static void nsim_rq_timer_init(struct nsim_rq *rq) 480 { 481 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, 482 HRTIMER_MODE_REL); 483 } 484 485 static void nsim_enable_napi(struct netdevsim *ns) 486 { 487 struct net_device *dev = ns->netdev; 488 int i; 489 490 for (i = 0; i < dev->num_rx_queues; i++) { 491 struct nsim_rq *rq = ns->rq[i]; 492 493 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 494 napi_enable_locked(&rq->napi); 495 } 496 } 497 498 static int nsim_open(struct net_device *dev) 499 { 500 struct netdevsim *ns = netdev_priv(dev); 501 struct netdevsim *peer; 502 int err; 503 504 netdev_assert_locked(dev); 505 506 err = nsim_init_napi(ns); 507 if (err) 508 return err; 509 510 nsim_enable_napi(ns); 511 512 peer = rtnl_dereference(ns->peer); 513 if (peer && netif_running(peer->netdev)) { 514 netif_carrier_on(dev); 515 netif_carrier_on(peer->netdev); 516 } 517 518 return 0; 519 } 520 521 static void nsim_del_napi(struct netdevsim *ns) 522 { 523 struct net_device *dev = ns->netdev; 524 int i; 525 526 for (i = 0; i < dev->num_rx_queues; i++) { 527 struct nsim_rq *rq = ns->rq[i]; 528 529 napi_disable_locked(&rq->napi); 530 __netif_napi_del_locked(&rq->napi); 531 } 532 synchronize_net(); 533 534 for (i = 0; i < dev->num_rx_queues; i++) { 535 page_pool_destroy(ns->rq[i]->page_pool); 536 ns->rq[i]->page_pool = NULL; 537 } 538 } 539 540 static int nsim_stop(struct net_device *dev) 541 { 542 struct netdevsim *ns = netdev_priv(dev); 543 struct netdevsim *peer; 544 545 netdev_assert_locked(dev); 546 547 netif_carrier_off(dev); 548 peer = rtnl_dereference(ns->peer); 549 if (peer) 550 netif_carrier_off(peer->netdev); 551 552 nsim_del_napi(ns); 553 554 return 0; 555 } 556 557 static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 558 { 559 struct netdevsim *ns = netdev_priv(dev); 560 561 if (vid >= VLAN_N_VID) 562 return -EINVAL; 563 564 if (proto == htons(ETH_P_8021Q)) 565 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag)); 566 else if (proto == htons(ETH_P_8021AD)) 567 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag)); 568 569 return 0; 570 } 571 572 static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 573 { 574 struct netdevsim *ns = netdev_priv(dev); 575 576 if (vid >= VLAN_N_VID) 577 return -EINVAL; 578 579 if (proto == htons(ETH_P_8021Q)) 580 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag)); 581 else if (proto == htons(ETH_P_8021AD)) 582 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag)); 583 584 return 0; 585 } 586 587 static int nsim_shaper_set(struct net_shaper_binding *binding, 588 const struct net_shaper *shaper, 589 struct netlink_ext_ack *extack) 590 { 591 return 0; 592 } 593 594 static int nsim_shaper_del(struct net_shaper_binding *binding, 595 const struct net_shaper_handle *handle, 596 struct netlink_ext_ack *extack) 597 { 598 return 0; 599 } 600 601 static int nsim_shaper_group(struct net_shaper_binding *binding, 602 int leaves_count, 603 const struct net_shaper *leaves, 604 const struct net_shaper *root, 605 struct netlink_ext_ack *extack) 606 { 607 return 0; 608 } 609 610 static void nsim_shaper_cap(struct net_shaper_binding *binding, 611 enum net_shaper_scope scope, 612 unsigned long *flags) 613 { 614 *flags = ULONG_MAX; 615 } 616 617 static const struct net_shaper_ops nsim_shaper_ops = { 618 .set = nsim_shaper_set, 619 .delete = nsim_shaper_del, 620 .group = nsim_shaper_group, 621 .capabilities = nsim_shaper_cap, 622 }; 623 624 static const struct net_device_ops nsim_netdev_ops = { 625 .ndo_start_xmit = nsim_start_xmit, 626 .ndo_set_rx_mode = nsim_set_rx_mode, 627 .ndo_set_mac_address = eth_mac_addr, 628 .ndo_validate_addr = eth_validate_addr, 629 .ndo_change_mtu = nsim_change_mtu, 630 .ndo_set_vf_mac = nsim_set_vf_mac, 631 .ndo_set_vf_vlan = nsim_set_vf_vlan, 632 .ndo_set_vf_rate = nsim_set_vf_rate, 633 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 634 .ndo_set_vf_trust = nsim_set_vf_trust, 635 .ndo_get_vf_config = nsim_get_vf_config, 636 .ndo_set_vf_link_state = nsim_set_vf_link_state, 637 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 638 .ndo_setup_tc = nsim_setup_tc, 639 .ndo_set_features = nsim_set_features, 640 .ndo_get_iflink = nsim_get_iflink, 641 .ndo_bpf = nsim_bpf, 642 .ndo_open = nsim_open, 643 .ndo_stop = nsim_stop, 644 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 645 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 646 .net_shaper_ops = &nsim_shaper_ops, 647 }; 648 649 static const struct net_device_ops nsim_vf_netdev_ops = { 650 .ndo_start_xmit = nsim_start_xmit, 651 .ndo_set_rx_mode = nsim_set_rx_mode, 652 .ndo_set_mac_address = eth_mac_addr, 653 .ndo_validate_addr = eth_validate_addr, 654 .ndo_change_mtu = nsim_change_mtu, 655 .ndo_setup_tc = nsim_setup_tc, 656 .ndo_set_features = nsim_set_features, 657 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 658 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 659 }; 660 661 /* We don't have true per-queue stats, yet, so do some random fakery here. 662 * Only report stuff for queue 0. 663 */ 664 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 665 struct netdev_queue_stats_rx *stats) 666 { 667 struct rtnl_link_stats64 rtstats = {}; 668 669 if (!idx) 670 dev_get_stats(dev, &rtstats); 671 672 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 673 stats->bytes = rtstats.rx_bytes; 674 } 675 676 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 677 struct netdev_queue_stats_tx *stats) 678 { 679 struct rtnl_link_stats64 rtstats = {}; 680 681 if (!idx) 682 dev_get_stats(dev, &rtstats); 683 684 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 685 stats->bytes = rtstats.tx_bytes; 686 } 687 688 static void nsim_get_base_stats(struct net_device *dev, 689 struct netdev_queue_stats_rx *rx, 690 struct netdev_queue_stats_tx *tx) 691 { 692 struct rtnl_link_stats64 rtstats = {}; 693 694 dev_get_stats(dev, &rtstats); 695 696 rx->packets = !!rtstats.rx_packets; 697 rx->bytes = 0; 698 tx->packets = !!rtstats.tx_packets; 699 tx->bytes = 0; 700 } 701 702 static const struct netdev_stat_ops nsim_stat_ops = { 703 .get_queue_stats_tx = nsim_get_queue_stats_tx, 704 .get_queue_stats_rx = nsim_get_queue_stats_rx, 705 .get_base_stats = nsim_get_base_stats, 706 }; 707 708 static struct nsim_rq *nsim_queue_alloc(void) 709 { 710 struct nsim_rq *rq; 711 712 rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT); 713 if (!rq) 714 return NULL; 715 716 skb_queue_head_init(&rq->skb_queue); 717 nsim_rq_timer_init(rq); 718 return rq; 719 } 720 721 static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq) 722 { 723 hrtimer_cancel(&rq->napi_timer); 724 725 if (rq->skb_queue.qlen) { 726 local_bh_disable(); 727 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); 728 local_bh_enable(); 729 } 730 731 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 732 kfree(rq); 733 } 734 735 /* Queue reset mode is controlled by ns->rq_reset_mode. 736 * - normal - new NAPI new pool (old NAPI enabled when new added) 737 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 738 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 739 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 740 */ 741 struct nsim_queue_mem { 742 struct nsim_rq *rq; 743 struct page_pool *pp; 744 }; 745 746 static int 747 nsim_queue_mem_alloc(struct net_device *dev, 748 struct netdev_queue_config *qcfg, 749 void *per_queue_mem, int idx) 750 { 751 struct nsim_queue_mem *qmem = per_queue_mem; 752 struct netdevsim *ns = netdev_priv(dev); 753 int err; 754 755 if (ns->rq_reset_mode > 3) 756 return -EINVAL; 757 758 if (ns->rq_reset_mode == 1) { 759 if (!netif_running(ns->netdev)) 760 return -ENETDOWN; 761 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 762 } 763 764 qmem->rq = nsim_queue_alloc(); 765 if (!qmem->rq) 766 return -ENOMEM; 767 768 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 769 if (err) 770 goto err_free; 771 772 if (!ns->rq_reset_mode) 773 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 774 idx); 775 776 return 0; 777 778 err_free: 779 nsim_queue_free(dev, qmem->rq); 780 return err; 781 } 782 783 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 784 { 785 struct nsim_queue_mem *qmem = per_queue_mem; 786 struct netdevsim *ns = netdev_priv(dev); 787 788 page_pool_destroy(qmem->pp); 789 if (qmem->rq) { 790 if (!ns->rq_reset_mode) 791 netif_napi_del_locked(&qmem->rq->napi); 792 page_pool_destroy(qmem->rq->page_pool); 793 nsim_queue_free(dev, qmem->rq); 794 } 795 } 796 797 static int 798 nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, 799 void *per_queue_mem, int idx) 800 { 801 struct nsim_queue_mem *qmem = per_queue_mem; 802 struct netdevsim *ns = netdev_priv(dev); 803 804 netdev_assert_locked(dev); 805 806 if (ns->rq_reset_mode == 1) { 807 ns->rq[idx]->page_pool = qmem->pp; 808 napi_enable_locked(&ns->rq[idx]->napi); 809 return 0; 810 } 811 812 /* netif_napi_add()/_del() should normally be called from alloc/free, 813 * here we want to test various call orders. 814 */ 815 if (ns->rq_reset_mode == 2) { 816 netif_napi_del_locked(&ns->rq[idx]->napi); 817 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 818 idx); 819 } else if (ns->rq_reset_mode == 3) { 820 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 821 idx); 822 netif_napi_del_locked(&ns->rq[idx]->napi); 823 } 824 825 ns->rq[idx] = qmem->rq; 826 napi_enable_locked(&ns->rq[idx]->napi); 827 828 return 0; 829 } 830 831 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 832 { 833 struct nsim_queue_mem *qmem = per_queue_mem; 834 struct netdevsim *ns = netdev_priv(dev); 835 836 netdev_assert_locked(dev); 837 838 napi_disable_locked(&ns->rq[idx]->napi); 839 840 if (ns->rq_reset_mode == 1) { 841 qmem->pp = ns->rq[idx]->page_pool; 842 page_pool_disable_direct_recycling(qmem->pp); 843 } else { 844 qmem->rq = ns->rq[idx]; 845 } 846 847 return 0; 848 } 849 850 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 851 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 852 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 853 .ndo_queue_mem_free = nsim_queue_mem_free, 854 .ndo_queue_start = nsim_queue_start, 855 .ndo_queue_stop = nsim_queue_stop, 856 }; 857 858 static ssize_t 859 nsim_qreset_write(struct file *file, const char __user *data, 860 size_t count, loff_t *ppos) 861 { 862 struct netdevsim *ns = file->private_data; 863 unsigned int queue, mode; 864 char buf[32]; 865 ssize_t ret; 866 867 if (count >= sizeof(buf)) 868 return -EINVAL; 869 if (copy_from_user(buf, data, count)) 870 return -EFAULT; 871 buf[count] = '\0'; 872 873 ret = sscanf(buf, "%u %u", &queue, &mode); 874 if (ret != 2) 875 return -EINVAL; 876 877 netdev_lock(ns->netdev); 878 if (queue >= ns->netdev->real_num_rx_queues) { 879 ret = -EINVAL; 880 goto exit_unlock; 881 } 882 883 ns->rq_reset_mode = mode; 884 ret = netdev_rx_queue_restart(ns->netdev, queue); 885 ns->rq_reset_mode = 0; 886 if (ret) 887 goto exit_unlock; 888 889 ret = count; 890 exit_unlock: 891 netdev_unlock(ns->netdev); 892 return ret; 893 } 894 895 static const struct file_operations nsim_qreset_fops = { 896 .open = simple_open, 897 .write = nsim_qreset_write, 898 .owner = THIS_MODULE, 899 }; 900 901 static ssize_t 902 nsim_pp_hold_read(struct file *file, char __user *data, 903 size_t count, loff_t *ppos) 904 { 905 struct netdevsim *ns = file->private_data; 906 char buf[3] = "n\n"; 907 908 if (ns->page) 909 buf[0] = 'y'; 910 911 return simple_read_from_buffer(data, count, ppos, buf, 2); 912 } 913 914 static ssize_t 915 nsim_pp_hold_write(struct file *file, const char __user *data, 916 size_t count, loff_t *ppos) 917 { 918 struct netdevsim *ns = file->private_data; 919 ssize_t ret; 920 bool val; 921 922 ret = kstrtobool_from_user(data, count, &val); 923 if (ret) 924 return ret; 925 926 rtnl_lock(); 927 ret = count; 928 if (val == !!ns->page) 929 goto exit; 930 931 if (!netif_running(ns->netdev) && val) { 932 ret = -ENETDOWN; 933 } else if (val) { 934 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 935 if (!ns->page) 936 ret = -ENOMEM; 937 } else { 938 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 939 ns->page, false); 940 ns->page = NULL; 941 } 942 943 exit: 944 rtnl_unlock(); 945 return ret; 946 } 947 948 static const struct file_operations nsim_pp_hold_fops = { 949 .open = simple_open, 950 .read = nsim_pp_hold_read, 951 .write = nsim_pp_hold_write, 952 .llseek = generic_file_llseek, 953 .owner = THIS_MODULE, 954 }; 955 956 static int nsim_vlan_show(struct seq_file *s, void *data) 957 { 958 struct netdevsim *ns = s->private; 959 int vid; 960 961 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 962 seq_printf(s, "ctag %d\n", vid); 963 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 964 seq_printf(s, "stag %d\n", vid); 965 966 return 0; 967 } 968 DEFINE_SHOW_ATTRIBUTE(nsim_vlan); 969 970 static void nsim_setup(struct net_device *dev) 971 { 972 ether_setup(dev); 973 eth_hw_addr_random(dev); 974 975 dev->flags &= ~IFF_MULTICAST; 976 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 977 dev->features |= NETIF_F_HIGHDMA | 978 NETIF_F_SG | 979 NETIF_F_FRAGLIST | 980 NETIF_F_HW_CSUM | 981 NETIF_F_LRO | 982 NETIF_F_TSO | 983 NETIF_F_HW_VLAN_CTAG_FILTER | 984 NETIF_F_HW_VLAN_STAG_FILTER; 985 dev->hw_features |= NETIF_F_HW_TC | 986 NETIF_F_SG | 987 NETIF_F_FRAGLIST | 988 NETIF_F_HW_CSUM | 989 NETIF_F_LRO | 990 NETIF_F_TSO | 991 NETIF_F_LOOPBACK | 992 NETIF_F_HW_VLAN_CTAG_FILTER | 993 NETIF_F_HW_VLAN_STAG_FILTER; 994 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; 995 dev->max_mtu = ETH_MAX_MTU; 996 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD; 997 } 998 999 static int nsim_queue_init(struct netdevsim *ns) 1000 { 1001 struct net_device *dev = ns->netdev; 1002 int i; 1003 1004 ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT); 1005 if (!ns->rq) 1006 return -ENOMEM; 1007 1008 for (i = 0; i < dev->num_rx_queues; i++) { 1009 ns->rq[i] = nsim_queue_alloc(); 1010 if (!ns->rq[i]) 1011 goto err_free_prev; 1012 } 1013 1014 return 0; 1015 1016 err_free_prev: 1017 while (i--) 1018 kfree(ns->rq[i]); 1019 kfree(ns->rq); 1020 return -ENOMEM; 1021 } 1022 1023 static void nsim_queue_uninit(struct netdevsim *ns) 1024 { 1025 struct net_device *dev = ns->netdev; 1026 int i; 1027 1028 for (i = 0; i < dev->num_rx_queues; i++) 1029 nsim_queue_free(dev, ns->rq[i]); 1030 1031 kfree(ns->rq); 1032 ns->rq = NULL; 1033 } 1034 1035 static int nsim_init_netdevsim(struct netdevsim *ns) 1036 { 1037 struct netdevsim *peer; 1038 struct mock_phc *phc; 1039 int err; 1040 1041 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 1042 if (IS_ERR(phc)) 1043 return PTR_ERR(phc); 1044 1045 ns->phc = phc; 1046 ns->netdev->netdev_ops = &nsim_netdev_ops; 1047 ns->netdev->stat_ops = &nsim_stat_ops; 1048 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 1049 netdev_lockdep_set_classes(ns->netdev); 1050 1051 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 1052 if (err) 1053 goto err_phc_destroy; 1054 1055 rtnl_lock(); 1056 err = nsim_queue_init(ns); 1057 if (err) 1058 goto err_utn_destroy; 1059 1060 err = nsim_bpf_init(ns); 1061 if (err) 1062 goto err_rq_destroy; 1063 1064 nsim_macsec_init(ns); 1065 nsim_ipsec_init(ns); 1066 1067 err = register_netdevice(ns->netdev); 1068 if (err) 1069 goto err_ipsec_teardown; 1070 rtnl_unlock(); 1071 1072 err = nsim_psp_init(ns); 1073 if (err) 1074 goto err_unregister_netdev; 1075 1076 if (IS_ENABLED(CONFIG_DEBUG_NET)) { 1077 ns->nb.notifier_call = netdev_debug_event; 1078 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1079 &ns->nn)) 1080 ns->nb.notifier_call = NULL; 1081 } 1082 1083 return 0; 1084 1085 err_unregister_netdev: 1086 rtnl_lock(); 1087 peer = rtnl_dereference(ns->peer); 1088 if (peer) 1089 RCU_INIT_POINTER(peer->peer, NULL); 1090 RCU_INIT_POINTER(ns->peer, NULL); 1091 unregister_netdevice(ns->netdev); 1092 err_ipsec_teardown: 1093 nsim_ipsec_teardown(ns); 1094 nsim_macsec_teardown(ns); 1095 nsim_bpf_uninit(ns); 1096 err_rq_destroy: 1097 nsim_queue_uninit(ns); 1098 err_utn_destroy: 1099 rtnl_unlock(); 1100 nsim_udp_tunnels_info_destroy(ns->netdev); 1101 err_phc_destroy: 1102 mock_phc_destroy(ns->phc); 1103 return err; 1104 } 1105 1106 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 1107 { 1108 int err; 1109 1110 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 1111 rtnl_lock(); 1112 err = register_netdevice(ns->netdev); 1113 rtnl_unlock(); 1114 return err; 1115 } 1116 1117 static void nsim_exit_netdevsim(struct netdevsim *ns) 1118 { 1119 nsim_udp_tunnels_info_destroy(ns->netdev); 1120 mock_phc_destroy(ns->phc); 1121 } 1122 1123 struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, 1124 struct nsim_dev_port *nsim_dev_port, 1125 u8 perm_addr[ETH_ALEN]) 1126 { 1127 struct net_device *dev; 1128 struct netdevsim *ns; 1129 int err; 1130 1131 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1132 nsim_dev->nsim_bus_dev->num_queues); 1133 if (!dev) 1134 return ERR_PTR(-ENOMEM); 1135 1136 if (perm_addr) 1137 memcpy(dev->perm_addr, perm_addr, ETH_ALEN); 1138 1139 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1140 ns = netdev_priv(dev); 1141 ns->netdev = dev; 1142 ns->nsim_dev = nsim_dev; 1143 ns->nsim_dev_port = nsim_dev_port; 1144 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1145 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1146 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1147 nsim_ethtool_init(ns); 1148 if (nsim_dev_port_is_pf(nsim_dev_port)) 1149 err = nsim_init_netdevsim(ns); 1150 else 1151 err = nsim_init_netdevsim_vf(ns); 1152 if (err) 1153 goto err_free_netdev; 1154 1155 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1156 ns, &nsim_pp_hold_fops); 1157 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1158 nsim_dev_port->ddir, ns, 1159 &nsim_qreset_fops); 1160 ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir, 1161 ns, &nsim_vlan_fops); 1162 return ns; 1163 1164 err_free_netdev: 1165 free_netdev(dev); 1166 return ERR_PTR(err); 1167 } 1168 1169 void nsim_destroy(struct netdevsim *ns) 1170 { 1171 struct net_device *dev = ns->netdev; 1172 struct netdevsim *peer; 1173 u16 vid; 1174 1175 debugfs_remove(ns->vlan_dfs); 1176 debugfs_remove(ns->qr_dfs); 1177 debugfs_remove(ns->pp_dfs); 1178 1179 if (ns->nb.notifier_call) 1180 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1181 &ns->nn); 1182 1183 nsim_psp_uninit(ns); 1184 1185 rtnl_lock(); 1186 peer = rtnl_dereference(ns->peer); 1187 if (peer) 1188 RCU_INIT_POINTER(peer->peer, NULL); 1189 RCU_INIT_POINTER(ns->peer, NULL); 1190 unregister_netdevice(dev); 1191 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1192 nsim_macsec_teardown(ns); 1193 nsim_ipsec_teardown(ns); 1194 nsim_bpf_uninit(ns); 1195 nsim_queue_uninit(ns); 1196 } 1197 rtnl_unlock(); 1198 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1199 nsim_exit_netdevsim(ns); 1200 1201 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 1202 WARN_ON_ONCE(1); 1203 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 1204 WARN_ON_ONCE(1); 1205 1206 /* Put this intentionally late to exercise the orphaning path */ 1207 if (ns->page) { 1208 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 1209 ns->page, false); 1210 ns->page = NULL; 1211 } 1212 1213 free_netdev(dev); 1214 } 1215 1216 bool netdev_is_nsim(struct net_device *dev) 1217 { 1218 return dev->netdev_ops == &nsim_netdev_ops; 1219 } 1220 1221 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1222 struct netlink_ext_ack *extack) 1223 { 1224 NL_SET_ERR_MSG_MOD(extack, 1225 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1226 return -EOPNOTSUPP; 1227 } 1228 1229 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1230 .kind = DRV_NAME, 1231 .validate = nsim_validate, 1232 }; 1233 1234 static int __init nsim_module_init(void) 1235 { 1236 int err; 1237 1238 err = nsim_dev_init(); 1239 if (err) 1240 return err; 1241 1242 err = nsim_bus_init(); 1243 if (err) 1244 goto err_dev_exit; 1245 1246 err = rtnl_link_register(&nsim_link_ops); 1247 if (err) 1248 goto err_bus_exit; 1249 1250 return 0; 1251 1252 err_bus_exit: 1253 nsim_bus_exit(); 1254 err_dev_exit: 1255 nsim_dev_exit(); 1256 return err; 1257 } 1258 1259 static void __exit nsim_module_exit(void) 1260 { 1261 rtnl_link_unregister(&nsim_link_ops); 1262 nsim_bus_exit(); 1263 nsim_dev_exit(); 1264 } 1265 1266 module_init(nsim_module_init); 1267 module_exit(nsim_module_exit); 1268 MODULE_LICENSE("GPL"); 1269 MODULE_DESCRIPTION("Simulated networking device for testing"); 1270 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1271