1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 #include <net/busy_poll.h> 33 34 #include "netdevsim.h" 35 36 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 37 38 #define NSIM_RING_SIZE 256 39 40 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq) 41 { 42 struct netdevsim *ns = netdev_priv(dev); 43 struct net_device *peer_dev; 44 struct netdevsim *peer_ns; 45 struct netdev_queue *txq; 46 u16 idx; 47 48 idx = rq->napi.index; 49 rcu_read_lock(); 50 peer_ns = rcu_dereference(ns->peer); 51 if (!peer_ns) 52 goto out; 53 54 /* TX device */ 55 peer_dev = peer_ns->netdev; 56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues) 57 goto out; 58 59 txq = netdev_get_tx_queue(peer_dev, idx); 60 if (!netif_tx_queue_stopped(txq)) 61 goto out; 62 63 netif_tx_wake_queue(txq); 64 out: 65 rcu_read_unlock(); 66 } 67 68 static void nsim_stop_tx_queue(struct net_device *tx_dev, 69 struct net_device *rx_dev, 70 struct nsim_rq *rq, 71 u16 idx) 72 { 73 /* If different queues size, do not stop, since it is not 74 * easy to find which TX queue is mapped here 75 */ 76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) 77 return; 78 79 /* rq is the queue on the receive side */ 80 netif_subqueue_try_stop(tx_dev, idx, 81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), 82 NSIM_RING_SIZE / 2); 83 } 84 85 static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, 86 struct nsim_rq *rq, struct sk_buff *skb) 87 { 88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 89 dev_kfree_skb_any(skb); 90 return NET_RX_DROP; 91 } 92 93 skb_queue_tail(&rq->skb_queue, skb); 94 95 /* Stop the peer TX queue avoiding dropping packets later */ 96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) 97 nsim_stop_tx_queue(tx_dev, rx_dev, rq, 98 skb_get_queue_mapping(skb)); 99 100 return NET_RX_SUCCESS; 101 } 102 103 static int nsim_forward_skb(struct net_device *tx_dev, 104 struct net_device *rx_dev, 105 struct sk_buff *skb, 106 struct nsim_rq *rq, 107 struct skb_ext *psp_ext) 108 { 109 int ret; 110 111 ret = __dev_forward_skb(rx_dev, skb); 112 if (ret) { 113 if (psp_ext) 114 __skb_ext_put(psp_ext); 115 return ret; 116 } 117 118 nsim_psp_handle_ext(skb, psp_ext); 119 120 return nsim_napi_rx(tx_dev, rx_dev, rq, skb); 121 } 122 123 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 124 { 125 struct netdevsim *ns = netdev_priv(dev); 126 struct skb_ext *psp_ext = NULL; 127 struct net_device *peer_dev; 128 unsigned int len = skb->len; 129 struct netdevsim *peer_ns; 130 struct netdev_config *cfg; 131 struct nsim_rq *rq; 132 int rxq; 133 int dr; 134 135 rcu_read_lock(); 136 if (!nsim_ipsec_tx(ns, skb)) 137 goto out_drop_any; 138 139 /* Check if loopback mode is enabled */ 140 if (dev->features & NETIF_F_LOOPBACK) { 141 peer_ns = ns; 142 peer_dev = dev; 143 } else { 144 peer_ns = rcu_dereference(ns->peer); 145 if (!peer_ns) 146 goto out_drop_any; 147 peer_dev = peer_ns->netdev; 148 } 149 150 dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext); 151 if (dr) 152 goto out_drop_free; 153 154 rxq = skb_get_queue_mapping(skb); 155 if (rxq >= peer_dev->num_rx_queues) 156 rxq = rxq % peer_dev->num_rx_queues; 157 rq = peer_ns->rq[rxq]; 158 159 cfg = peer_dev->cfg; 160 if (skb_is_nonlinear(skb) && 161 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 162 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 163 cfg->hds_thresh > len))) 164 skb_linearize(skb); 165 166 skb_tx_timestamp(skb); 167 if (unlikely(nsim_forward_skb(dev, peer_dev, 168 skb, rq, psp_ext) == NET_RX_DROP)) 169 goto out_drop_cnt; 170 171 if (!hrtimer_active(&rq->napi_timer)) 172 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 173 174 rcu_read_unlock(); 175 dev_dstats_tx_add(dev, len); 176 return NETDEV_TX_OK; 177 178 out_drop_any: 179 dr = SKB_DROP_REASON_NOT_SPECIFIED; 180 out_drop_free: 181 kfree_skb_reason(skb, dr); 182 out_drop_cnt: 183 rcu_read_unlock(); 184 dev_dstats_tx_dropped(dev); 185 return NETDEV_TX_OK; 186 } 187 188 static int nsim_set_rx_mode(struct net_device *dev, 189 struct netdev_hw_addr_list *uc, 190 struct netdev_hw_addr_list *mc) 191 { 192 return 0; 193 } 194 195 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 196 { 197 struct netdevsim *ns = netdev_priv(dev); 198 199 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 200 new_mtu > NSIM_XDP_MAX_MTU) 201 return -EBUSY; 202 203 WRITE_ONCE(dev->mtu, new_mtu); 204 205 return 0; 206 } 207 208 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 209 { 210 struct netdevsim *ns = netdev_priv(dev); 211 struct nsim_dev *nsim_dev = ns->nsim_dev; 212 213 /* Only refuse multicast addresses, zero address can mean unset/any. */ 214 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 215 return -EINVAL; 216 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 217 218 return 0; 219 } 220 221 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 222 u16 vlan, u8 qos, __be16 vlan_proto) 223 { 224 struct netdevsim *ns = netdev_priv(dev); 225 struct nsim_dev *nsim_dev = ns->nsim_dev; 226 227 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 228 return -EINVAL; 229 230 nsim_dev->vfconfigs[vf].vlan = vlan; 231 nsim_dev->vfconfigs[vf].qos = qos; 232 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 233 234 return 0; 235 } 236 237 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 238 { 239 struct netdevsim *ns = netdev_priv(dev); 240 struct nsim_dev *nsim_dev = ns->nsim_dev; 241 242 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 243 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 244 return -EOPNOTSUPP; 245 } 246 247 if (vf >= nsim_dev_get_vfs(nsim_dev)) 248 return -EINVAL; 249 250 nsim_dev->vfconfigs[vf].min_tx_rate = min; 251 nsim_dev->vfconfigs[vf].max_tx_rate = max; 252 253 return 0; 254 } 255 256 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 257 { 258 struct netdevsim *ns = netdev_priv(dev); 259 struct nsim_dev *nsim_dev = ns->nsim_dev; 260 261 if (vf >= nsim_dev_get_vfs(nsim_dev)) 262 return -EINVAL; 263 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 264 265 return 0; 266 } 267 268 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 269 { 270 struct netdevsim *ns = netdev_priv(dev); 271 struct nsim_dev *nsim_dev = ns->nsim_dev; 272 273 if (vf >= nsim_dev_get_vfs(nsim_dev)) 274 return -EINVAL; 275 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 276 277 return 0; 278 } 279 280 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 281 { 282 struct netdevsim *ns = netdev_priv(dev); 283 struct nsim_dev *nsim_dev = ns->nsim_dev; 284 285 if (vf >= nsim_dev_get_vfs(nsim_dev)) 286 return -EINVAL; 287 nsim_dev->vfconfigs[vf].trusted = val; 288 289 return 0; 290 } 291 292 static int 293 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 294 { 295 struct netdevsim *ns = netdev_priv(dev); 296 struct nsim_dev *nsim_dev = ns->nsim_dev; 297 298 if (vf >= nsim_dev_get_vfs(nsim_dev)) 299 return -EINVAL; 300 301 ivi->vf = vf; 302 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 303 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 304 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 305 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 306 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 307 ivi->qos = nsim_dev->vfconfigs[vf].qos; 308 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 309 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 310 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 311 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 312 313 return 0; 314 } 315 316 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 317 { 318 struct netdevsim *ns = netdev_priv(dev); 319 struct nsim_dev *nsim_dev = ns->nsim_dev; 320 321 if (vf >= nsim_dev_get_vfs(nsim_dev)) 322 return -EINVAL; 323 324 switch (state) { 325 case IFLA_VF_LINK_STATE_AUTO: 326 case IFLA_VF_LINK_STATE_ENABLE: 327 case IFLA_VF_LINK_STATE_DISABLE: 328 break; 329 default: 330 return -EINVAL; 331 } 332 333 nsim_dev->vfconfigs[vf].link_state = state; 334 335 return 0; 336 } 337 338 static int 339 nsim_set_features(struct net_device *dev, netdev_features_t features) 340 { 341 struct netdevsim *ns = netdev_priv(dev); 342 343 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 344 return nsim_bpf_disable_tc(ns); 345 346 return 0; 347 } 348 349 static int nsim_get_iflink(const struct net_device *dev) 350 { 351 struct netdevsim *nsim, *peer; 352 int iflink; 353 354 nsim = netdev_priv(dev); 355 356 rcu_read_lock(); 357 peer = rcu_dereference(nsim->peer); 358 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 359 READ_ONCE(dev->ifindex); 360 rcu_read_unlock(); 361 362 return iflink; 363 } 364 365 static int nsim_rcv(struct nsim_rq *rq, int budget) 366 { 367 struct net_device *dev = rq->napi.dev; 368 struct bpf_prog *xdp_prog; 369 struct netdevsim *ns; 370 struct sk_buff *skb; 371 unsigned int skblen; 372 int i, ret; 373 374 ns = netdev_priv(dev); 375 xdp_prog = READ_ONCE(ns->xdp.prog); 376 377 for (i = 0; i < budget; i++) { 378 if (skb_queue_empty(&rq->skb_queue)) 379 break; 380 381 skb = skb_dequeue(&rq->skb_queue); 382 383 if (xdp_prog) { 384 /* skb might be freed directly by XDP, save the len */ 385 skblen = skb->len; 386 387 if (skb->ip_summed == CHECKSUM_PARTIAL) 388 skb_checksum_help(skb); 389 ret = do_xdp_generic(xdp_prog, &skb); 390 if (ret != XDP_PASS) { 391 dev_dstats_rx_add(dev, skblen); 392 continue; 393 } 394 } 395 396 /* skb might be discard at netif_receive_skb, save the len */ 397 dev_dstats_rx_add(dev, skb->len); 398 napi_gro_receive(&rq->napi, skb); 399 } 400 401 nsim_start_peer_tx_queue(dev, rq); 402 return i; 403 } 404 405 static int nsim_poll(struct napi_struct *napi, int budget) 406 { 407 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 408 int done; 409 410 done = nsim_rcv(rq, budget); 411 if (done < budget) 412 napi_complete_done(napi, done); 413 414 return done; 415 } 416 417 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 418 { 419 struct page_pool_params params = { 420 .order = 0, 421 .pool_size = NSIM_RING_SIZE, 422 .nid = NUMA_NO_NODE, 423 .dev = &napi->dev->dev, 424 .napi = napi, 425 .dma_dir = DMA_BIDIRECTIONAL, 426 .netdev = napi->dev, 427 }; 428 struct page_pool *pool; 429 430 pool = page_pool_create(¶ms); 431 if (IS_ERR(pool)) 432 return PTR_ERR(pool); 433 434 *p = pool; 435 return 0; 436 } 437 438 static int nsim_init_napi(struct netdevsim *ns) 439 { 440 struct net_device *dev = ns->netdev; 441 struct nsim_rq *rq; 442 int err, i; 443 444 for (i = 0; i < dev->num_rx_queues; i++) { 445 rq = ns->rq[i]; 446 447 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 448 } 449 450 for (i = 0; i < dev->num_rx_queues; i++) { 451 rq = ns->rq[i]; 452 453 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 454 if (err) 455 goto err_pp_destroy; 456 } 457 458 return 0; 459 460 err_pp_destroy: 461 while (i--) { 462 page_pool_destroy(ns->rq[i]->page_pool); 463 ns->rq[i]->page_pool = NULL; 464 } 465 466 for (i = 0; i < dev->num_rx_queues; i++) 467 __netif_napi_del_locked(&ns->rq[i]->napi); 468 469 return err; 470 } 471 472 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 473 { 474 struct nsim_rq *rq; 475 476 rq = container_of(timer, struct nsim_rq, napi_timer); 477 napi_schedule(&rq->napi); 478 479 return HRTIMER_NORESTART; 480 } 481 482 static void nsim_rq_timer_init(struct nsim_rq *rq) 483 { 484 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, 485 HRTIMER_MODE_REL); 486 } 487 488 static void nsim_enable_napi(struct netdevsim *ns) 489 { 490 struct net_device *dev = ns->netdev; 491 int i; 492 493 for (i = 0; i < dev->num_rx_queues; i++) { 494 struct nsim_rq *rq = ns->rq[i]; 495 496 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 497 napi_enable_locked(&rq->napi); 498 } 499 } 500 501 static int nsim_open(struct net_device *dev) 502 { 503 struct netdevsim *ns = netdev_priv(dev); 504 struct netdevsim *peer; 505 int err; 506 507 netdev_assert_locked(dev); 508 509 err = nsim_init_napi(ns); 510 if (err) 511 return err; 512 513 nsim_enable_napi(ns); 514 515 peer = rtnl_dereference(ns->peer); 516 if (peer && netif_running(peer->netdev)) { 517 netif_carrier_on(dev); 518 netif_carrier_on(peer->netdev); 519 } 520 521 return 0; 522 } 523 524 static void nsim_del_napi(struct netdevsim *ns) 525 { 526 struct net_device *dev = ns->netdev; 527 int i; 528 529 for (i = 0; i < dev->num_rx_queues; i++) { 530 struct nsim_rq *rq = ns->rq[i]; 531 532 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL); 533 napi_disable_locked(&rq->napi); 534 __netif_napi_del_locked(&rq->napi); 535 } 536 synchronize_net(); 537 538 for (i = 0; i < dev->num_rx_queues; i++) { 539 page_pool_destroy(ns->rq[i]->page_pool); 540 ns->rq[i]->page_pool = NULL; 541 } 542 } 543 544 static int nsim_stop(struct net_device *dev) 545 { 546 struct netdevsim *ns = netdev_priv(dev); 547 struct netdevsim *peer; 548 549 netdev_assert_locked(dev); 550 551 netif_carrier_off(dev); 552 peer = rtnl_dereference(ns->peer); 553 if (peer) 554 netif_carrier_off(peer->netdev); 555 556 nsim_del_napi(ns); 557 558 return 0; 559 } 560 561 static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 562 { 563 struct netdevsim *ns = netdev_priv(dev); 564 565 if (vid >= VLAN_N_VID) 566 return -EINVAL; 567 568 if (proto == htons(ETH_P_8021Q)) 569 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag)); 570 else if (proto == htons(ETH_P_8021AD)) 571 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag)); 572 573 return 0; 574 } 575 576 static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 577 { 578 struct netdevsim *ns = netdev_priv(dev); 579 580 if (vid >= VLAN_N_VID) 581 return -EINVAL; 582 583 if (proto == htons(ETH_P_8021Q)) 584 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag)); 585 else if (proto == htons(ETH_P_8021AD)) 586 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag)); 587 588 return 0; 589 } 590 591 static int nsim_shaper_set(struct net_shaper_binding *binding, 592 const struct net_shaper *shaper, 593 struct netlink_ext_ack *extack) 594 { 595 return 0; 596 } 597 598 static int nsim_shaper_del(struct net_shaper_binding *binding, 599 const struct net_shaper_handle *handle, 600 struct netlink_ext_ack *extack) 601 { 602 return 0; 603 } 604 605 static int nsim_shaper_group(struct net_shaper_binding *binding, 606 int leaves_count, 607 const struct net_shaper *leaves, 608 const struct net_shaper *root, 609 struct netlink_ext_ack *extack) 610 { 611 return 0; 612 } 613 614 static void nsim_shaper_cap(struct net_shaper_binding *binding, 615 enum net_shaper_scope scope, 616 unsigned long *flags) 617 { 618 *flags = ULONG_MAX; 619 } 620 621 static const struct net_shaper_ops nsim_shaper_ops = { 622 .set = nsim_shaper_set, 623 .delete = nsim_shaper_del, 624 .group = nsim_shaper_group, 625 .capabilities = nsim_shaper_cap, 626 }; 627 628 static const struct net_device_ops nsim_netdev_ops = { 629 .ndo_start_xmit = nsim_start_xmit, 630 .ndo_set_rx_mode_async = nsim_set_rx_mode, 631 .ndo_set_mac_address = eth_mac_addr, 632 .ndo_validate_addr = eth_validate_addr, 633 .ndo_change_mtu = nsim_change_mtu, 634 .ndo_set_vf_mac = nsim_set_vf_mac, 635 .ndo_set_vf_vlan = nsim_set_vf_vlan, 636 .ndo_set_vf_rate = nsim_set_vf_rate, 637 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 638 .ndo_set_vf_trust = nsim_set_vf_trust, 639 .ndo_get_vf_config = nsim_get_vf_config, 640 .ndo_set_vf_link_state = nsim_set_vf_link_state, 641 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 642 .ndo_setup_tc = nsim_setup_tc, 643 .ndo_set_features = nsim_set_features, 644 .ndo_get_iflink = nsim_get_iflink, 645 .ndo_bpf = nsim_bpf, 646 .ndo_open = nsim_open, 647 .ndo_stop = nsim_stop, 648 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 649 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 650 .net_shaper_ops = &nsim_shaper_ops, 651 }; 652 653 static const struct net_device_ops nsim_vf_netdev_ops = { 654 .ndo_start_xmit = nsim_start_xmit, 655 .ndo_set_rx_mode_async = nsim_set_rx_mode, 656 .ndo_set_mac_address = eth_mac_addr, 657 .ndo_validate_addr = eth_validate_addr, 658 .ndo_change_mtu = nsim_change_mtu, 659 .ndo_setup_tc = nsim_setup_tc, 660 .ndo_set_features = nsim_set_features, 661 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 662 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 663 }; 664 665 /* We don't have true per-queue stats, yet, so do some random fakery here. 666 * Only report stuff for queue 0. 667 */ 668 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 669 struct netdev_queue_stats_rx *stats) 670 { 671 struct rtnl_link_stats64 rtstats = {}; 672 673 if (!idx) 674 dev_get_stats(dev, &rtstats); 675 676 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 677 stats->bytes = rtstats.rx_bytes; 678 } 679 680 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 681 struct netdev_queue_stats_tx *stats) 682 { 683 struct rtnl_link_stats64 rtstats = {}; 684 685 if (!idx) 686 dev_get_stats(dev, &rtstats); 687 688 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 689 stats->bytes = rtstats.tx_bytes; 690 } 691 692 static void nsim_get_base_stats(struct net_device *dev, 693 struct netdev_queue_stats_rx *rx, 694 struct netdev_queue_stats_tx *tx) 695 { 696 struct rtnl_link_stats64 rtstats = {}; 697 698 dev_get_stats(dev, &rtstats); 699 700 rx->packets = !!rtstats.rx_packets; 701 rx->bytes = 0; 702 tx->packets = !!rtstats.tx_packets; 703 tx->bytes = 0; 704 } 705 706 static const struct netdev_stat_ops nsim_stat_ops = { 707 .get_queue_stats_tx = nsim_get_queue_stats_tx, 708 .get_queue_stats_rx = nsim_get_queue_stats_rx, 709 .get_base_stats = nsim_get_base_stats, 710 }; 711 712 static struct nsim_rq *nsim_queue_alloc(void) 713 { 714 struct nsim_rq *rq; 715 716 rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT); 717 if (!rq) 718 return NULL; 719 720 skb_queue_head_init(&rq->skb_queue); 721 nsim_rq_timer_init(rq); 722 return rq; 723 } 724 725 static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq) 726 { 727 hrtimer_cancel(&rq->napi_timer); 728 729 if (rq->skb_queue.qlen) { 730 local_bh_disable(); 731 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); 732 local_bh_enable(); 733 } 734 735 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 736 kfree(rq); 737 } 738 739 /* Queue reset mode is controlled by ns->rq_reset_mode. 740 * - normal - new NAPI new pool (old NAPI enabled when new added) 741 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 742 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 743 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 744 */ 745 struct nsim_queue_mem { 746 struct nsim_rq *rq; 747 struct page_pool *pp; 748 }; 749 750 static int 751 nsim_queue_mem_alloc(struct net_device *dev, 752 struct netdev_queue_config *qcfg, 753 void *per_queue_mem, int idx) 754 { 755 struct nsim_queue_mem *qmem = per_queue_mem; 756 struct netdevsim *ns = netdev_priv(dev); 757 int err; 758 759 if (ns->rq_reset_mode > 3) 760 return -EINVAL; 761 762 if (ns->rq_reset_mode == 1) { 763 if (!netif_running(ns->netdev)) 764 return -ENETDOWN; 765 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 766 } 767 768 qmem->rq = nsim_queue_alloc(); 769 if (!qmem->rq) 770 return -ENOMEM; 771 772 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 773 if (err) 774 goto err_free; 775 776 if (!ns->rq_reset_mode) 777 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 778 idx); 779 780 return 0; 781 782 err_free: 783 nsim_queue_free(dev, qmem->rq); 784 return err; 785 } 786 787 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 788 { 789 struct nsim_queue_mem *qmem = per_queue_mem; 790 struct netdevsim *ns = netdev_priv(dev); 791 792 page_pool_destroy(qmem->pp); 793 if (qmem->rq) { 794 if (!ns->rq_reset_mode) 795 netif_napi_del_locked(&qmem->rq->napi); 796 page_pool_destroy(qmem->rq->page_pool); 797 nsim_queue_free(dev, qmem->rq); 798 } 799 } 800 801 static int 802 nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, 803 void *per_queue_mem, int idx) 804 { 805 struct nsim_queue_mem *qmem = per_queue_mem; 806 struct netdevsim *ns = netdev_priv(dev); 807 808 netdev_assert_locked(dev); 809 810 if (ns->rq_reset_mode == 1) { 811 ns->rq[idx]->page_pool = qmem->pp; 812 napi_enable_locked(&ns->rq[idx]->napi); 813 return 0; 814 } 815 816 /* netif_napi_add()/_del() should normally be called from alloc/free, 817 * here we want to test various call orders. 818 */ 819 if (ns->rq_reset_mode == 2) { 820 netif_napi_del_locked(&ns->rq[idx]->napi); 821 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 822 idx); 823 } else if (ns->rq_reset_mode == 3) { 824 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 825 idx); 826 netif_napi_del_locked(&ns->rq[idx]->napi); 827 } 828 829 ns->rq[idx] = qmem->rq; 830 netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi); 831 napi_enable_locked(&ns->rq[idx]->napi); 832 833 return 0; 834 } 835 836 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 837 { 838 struct nsim_queue_mem *qmem = per_queue_mem; 839 struct netdevsim *ns = netdev_priv(dev); 840 841 netdev_assert_locked(dev); 842 843 napi_disable_locked(&ns->rq[idx]->napi); 844 845 if (ns->rq_reset_mode == 1) { 846 qmem->pp = ns->rq[idx]->page_pool; 847 page_pool_disable_direct_recycling(qmem->pp); 848 } else { 849 qmem->rq = ns->rq[idx]; 850 } 851 852 return 0; 853 } 854 855 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 856 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 857 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 858 .ndo_queue_mem_free = nsim_queue_mem_free, 859 .ndo_queue_start = nsim_queue_start, 860 .ndo_queue_stop = nsim_queue_stop, 861 }; 862 863 static ssize_t 864 nsim_qreset_write(struct file *file, const char __user *data, 865 size_t count, loff_t *ppos) 866 { 867 struct netdevsim *ns = file->private_data; 868 unsigned int queue, mode; 869 char buf[32]; 870 ssize_t ret; 871 872 if (count >= sizeof(buf)) 873 return -EINVAL; 874 if (copy_from_user(buf, data, count)) 875 return -EFAULT; 876 buf[count] = '\0'; 877 878 ret = sscanf(buf, "%u %u", &queue, &mode); 879 if (ret != 2) 880 return -EINVAL; 881 882 netdev_lock(ns->netdev); 883 if (queue >= ns->netdev->real_num_rx_queues) { 884 ret = -EINVAL; 885 goto exit_unlock; 886 } 887 888 ns->rq_reset_mode = mode; 889 ret = netdev_rx_queue_restart(ns->netdev, queue); 890 ns->rq_reset_mode = 0; 891 if (ret) 892 goto exit_unlock; 893 894 ret = count; 895 exit_unlock: 896 netdev_unlock(ns->netdev); 897 return ret; 898 } 899 900 static const struct file_operations nsim_qreset_fops = { 901 .open = simple_open, 902 .write = nsim_qreset_write, 903 .owner = THIS_MODULE, 904 }; 905 906 static ssize_t 907 nsim_pp_hold_read(struct file *file, char __user *data, 908 size_t count, loff_t *ppos) 909 { 910 struct netdevsim *ns = file->private_data; 911 char buf[3] = "n\n"; 912 913 if (ns->page) 914 buf[0] = 'y'; 915 916 return simple_read_from_buffer(data, count, ppos, buf, 2); 917 } 918 919 static ssize_t 920 nsim_pp_hold_write(struct file *file, const char __user *data, 921 size_t count, loff_t *ppos) 922 { 923 struct netdevsim *ns = file->private_data; 924 ssize_t ret; 925 bool val; 926 927 ret = kstrtobool_from_user(data, count, &val); 928 if (ret) 929 return ret; 930 931 rtnl_lock(); 932 ret = count; 933 if (val == !!ns->page) 934 goto exit; 935 936 if (!netif_running(ns->netdev) && val) { 937 ret = -ENETDOWN; 938 } else if (val) { 939 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 940 if (!ns->page) 941 ret = -ENOMEM; 942 } else { 943 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 944 ns->page, false); 945 ns->page = NULL; 946 } 947 948 exit: 949 rtnl_unlock(); 950 return ret; 951 } 952 953 static const struct file_operations nsim_pp_hold_fops = { 954 .open = simple_open, 955 .read = nsim_pp_hold_read, 956 .write = nsim_pp_hold_write, 957 .llseek = generic_file_llseek, 958 .owner = THIS_MODULE, 959 }; 960 961 static int nsim_vlan_show(struct seq_file *s, void *data) 962 { 963 struct netdevsim *ns = s->private; 964 int vid; 965 966 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 967 seq_printf(s, "ctag %d\n", vid); 968 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 969 seq_printf(s, "stag %d\n", vid); 970 971 return 0; 972 } 973 DEFINE_SHOW_ATTRIBUTE(nsim_vlan); 974 975 static void nsim_setup(struct net_device *dev) 976 { 977 ether_setup(dev); 978 eth_hw_addr_random(dev); 979 980 dev->flags &= ~IFF_MULTICAST; 981 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 982 dev->features |= NETIF_F_HIGHDMA | 983 NETIF_F_SG | 984 NETIF_F_FRAGLIST | 985 NETIF_F_HW_CSUM | 986 NETIF_F_LRO | 987 NETIF_F_TSO | 988 NETIF_F_HW_VLAN_CTAG_FILTER | 989 NETIF_F_HW_VLAN_STAG_FILTER; 990 dev->hw_features |= NETIF_F_HW_TC | 991 NETIF_F_SG | 992 NETIF_F_FRAGLIST | 993 NETIF_F_HW_CSUM | 994 NETIF_F_LRO | 995 NETIF_F_TSO | 996 NETIF_F_LOOPBACK | 997 NETIF_F_HW_VLAN_CTAG_FILTER | 998 NETIF_F_HW_VLAN_STAG_FILTER; 999 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; 1000 dev->max_mtu = ETH_MAX_MTU; 1001 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD; 1002 } 1003 1004 static int nsim_queue_init(struct netdevsim *ns) 1005 { 1006 struct net_device *dev = ns->netdev; 1007 int i; 1008 1009 ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT); 1010 if (!ns->rq) 1011 return -ENOMEM; 1012 1013 for (i = 0; i < dev->num_rx_queues; i++) { 1014 ns->rq[i] = nsim_queue_alloc(); 1015 if (!ns->rq[i]) 1016 goto err_free_prev; 1017 } 1018 1019 return 0; 1020 1021 err_free_prev: 1022 while (i--) 1023 kfree(ns->rq[i]); 1024 kfree(ns->rq); 1025 return -ENOMEM; 1026 } 1027 1028 static void nsim_queue_uninit(struct netdevsim *ns) 1029 { 1030 struct net_device *dev = ns->netdev; 1031 int i; 1032 1033 for (i = 0; i < dev->num_rx_queues; i++) 1034 nsim_queue_free(dev, ns->rq[i]); 1035 1036 kfree(ns->rq); 1037 ns->rq = NULL; 1038 } 1039 1040 static int nsim_init_netdevsim(struct netdevsim *ns) 1041 { 1042 struct netdevsim *peer; 1043 struct mock_phc *phc; 1044 int err; 1045 1046 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 1047 if (IS_ERR(phc)) 1048 return PTR_ERR(phc); 1049 1050 ns->phc = phc; 1051 ns->netdev->netdev_ops = &nsim_netdev_ops; 1052 ns->netdev->stat_ops = &nsim_stat_ops; 1053 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 1054 netdev_lockdep_set_classes(ns->netdev); 1055 1056 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 1057 if (err) 1058 goto err_phc_destroy; 1059 1060 rtnl_lock(); 1061 err = nsim_queue_init(ns); 1062 if (err) 1063 goto err_utn_destroy; 1064 1065 err = nsim_bpf_init(ns); 1066 if (err) 1067 goto err_rq_destroy; 1068 1069 nsim_macsec_init(ns); 1070 nsim_ipsec_init(ns); 1071 1072 err = register_netdevice(ns->netdev); 1073 if (err) 1074 goto err_ipsec_teardown; 1075 rtnl_unlock(); 1076 1077 err = nsim_psp_init(ns); 1078 if (err) 1079 goto err_unregister_netdev; 1080 1081 if (IS_ENABLED(CONFIG_DEBUG_NET)) { 1082 ns->nb.notifier_call = netdev_debug_event; 1083 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1084 &ns->nn)) 1085 ns->nb.notifier_call = NULL; 1086 } 1087 1088 return 0; 1089 1090 err_unregister_netdev: 1091 rtnl_lock(); 1092 peer = rtnl_dereference(ns->peer); 1093 if (peer) 1094 RCU_INIT_POINTER(peer->peer, NULL); 1095 RCU_INIT_POINTER(ns->peer, NULL); 1096 unregister_netdevice(ns->netdev); 1097 err_ipsec_teardown: 1098 nsim_ipsec_teardown(ns); 1099 nsim_macsec_teardown(ns); 1100 nsim_bpf_uninit(ns); 1101 err_rq_destroy: 1102 nsim_queue_uninit(ns); 1103 err_utn_destroy: 1104 rtnl_unlock(); 1105 nsim_udp_tunnels_info_destroy(ns->netdev); 1106 err_phc_destroy: 1107 mock_phc_destroy(ns->phc); 1108 return err; 1109 } 1110 1111 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 1112 { 1113 int err; 1114 1115 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 1116 rtnl_lock(); 1117 err = register_netdevice(ns->netdev); 1118 rtnl_unlock(); 1119 return err; 1120 } 1121 1122 static void nsim_exit_netdevsim(struct netdevsim *ns) 1123 { 1124 nsim_udp_tunnels_info_destroy(ns->netdev); 1125 mock_phc_destroy(ns->phc); 1126 } 1127 1128 struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, 1129 struct nsim_dev_port *nsim_dev_port, 1130 u8 perm_addr[ETH_ALEN]) 1131 { 1132 struct net_device *dev; 1133 struct netdevsim *ns; 1134 int err; 1135 1136 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1137 nsim_dev->nsim_bus_dev->num_queues); 1138 if (!dev) 1139 return ERR_PTR(-ENOMEM); 1140 1141 if (perm_addr) 1142 memcpy(dev->perm_addr, perm_addr, ETH_ALEN); 1143 1144 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1145 ns = netdev_priv(dev); 1146 ns->netdev = dev; 1147 ns->nsim_dev = nsim_dev; 1148 ns->nsim_dev_port = nsim_dev_port; 1149 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1150 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1151 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1152 nsim_ethtool_init(ns); 1153 if (nsim_dev_port_is_pf(nsim_dev_port)) 1154 err = nsim_init_netdevsim(ns); 1155 else 1156 err = nsim_init_netdevsim_vf(ns); 1157 if (err) 1158 goto err_free_netdev; 1159 1160 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1161 ns, &nsim_pp_hold_fops); 1162 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1163 nsim_dev_port->ddir, ns, 1164 &nsim_qreset_fops); 1165 ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir, 1166 ns, &nsim_vlan_fops); 1167 return ns; 1168 1169 err_free_netdev: 1170 nsim_ethtool_fini(ns); 1171 free_netdev(dev); 1172 return ERR_PTR(err); 1173 } 1174 1175 void nsim_destroy(struct netdevsim *ns) 1176 { 1177 struct net_device *dev = ns->netdev; 1178 struct netdevsim *peer; 1179 u16 vid; 1180 1181 debugfs_remove(ns->vlan_dfs); 1182 debugfs_remove(ns->qr_dfs); 1183 debugfs_remove(ns->pp_dfs); 1184 nsim_ethtool_fini(ns); 1185 1186 if (ns->nb.notifier_call) 1187 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1188 &ns->nn); 1189 1190 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1191 nsim_psp_uninit(ns); 1192 1193 rtnl_lock(); 1194 peer = rtnl_dereference(ns->peer); 1195 if (peer) 1196 RCU_INIT_POINTER(peer->peer, NULL); 1197 RCU_INIT_POINTER(ns->peer, NULL); 1198 unregister_netdevice(dev); 1199 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1200 nsim_macsec_teardown(ns); 1201 nsim_ipsec_teardown(ns); 1202 nsim_bpf_uninit(ns); 1203 nsim_queue_uninit(ns); 1204 } 1205 rtnl_unlock(); 1206 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1207 nsim_exit_netdevsim(ns); 1208 1209 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 1210 WARN_ON_ONCE(1); 1211 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 1212 WARN_ON_ONCE(1); 1213 1214 /* Put this intentionally late to exercise the orphaning path */ 1215 if (ns->page) { 1216 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 1217 ns->page, false); 1218 ns->page = NULL; 1219 } 1220 1221 free_netdev(dev); 1222 } 1223 1224 bool netdev_is_nsim(struct net_device *dev) 1225 { 1226 return dev->netdev_ops == &nsim_netdev_ops; 1227 } 1228 1229 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1230 struct netlink_ext_ack *extack) 1231 { 1232 NL_SET_ERR_MSG_MOD(extack, 1233 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1234 return -EOPNOTSUPP; 1235 } 1236 1237 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1238 .kind = DRV_NAME, 1239 .validate = nsim_validate, 1240 }; 1241 1242 static int __init nsim_module_init(void) 1243 { 1244 int err; 1245 1246 err = nsim_dev_init(); 1247 if (err) 1248 return err; 1249 1250 err = nsim_bus_init(); 1251 if (err) 1252 goto err_dev_exit; 1253 1254 err = rtnl_link_register(&nsim_link_ops); 1255 if (err) 1256 goto err_bus_exit; 1257 1258 return 0; 1259 1260 err_bus_exit: 1261 nsim_bus_exit(); 1262 err_dev_exit: 1263 nsim_dev_exit(); 1264 return err; 1265 } 1266 1267 static void __exit nsim_module_exit(void) 1268 { 1269 rtnl_link_unregister(&nsim_link_ops); 1270 nsim_bus_exit(); 1271 nsim_dev_exit(); 1272 } 1273 1274 module_init(nsim_module_init); 1275 module_exit(nsim_module_exit); 1276 MODULE_LICENSE("GPL"); 1277 MODULE_DESCRIPTION("Simulated networking device for testing"); 1278 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1279