1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 #include <net/busy_poll.h> 33 34 #include "netdevsim.h" 35 36 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 37 38 #define NSIM_RING_SIZE 256 39 40 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq) 41 { 42 struct netdevsim *ns = netdev_priv(dev); 43 struct net_device *peer_dev; 44 struct netdevsim *peer_ns; 45 struct netdev_queue *txq; 46 u16 idx; 47 48 idx = rq->napi.index; 49 rcu_read_lock(); 50 peer_ns = rcu_dereference(ns->peer); 51 if (!peer_ns) 52 goto out; 53 54 /* TX device */ 55 peer_dev = peer_ns->netdev; 56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues) 57 goto out; 58 59 txq = netdev_get_tx_queue(peer_dev, idx); 60 if (!netif_tx_queue_stopped(txq)) 61 goto out; 62 63 netif_tx_wake_queue(txq); 64 out: 65 rcu_read_unlock(); 66 } 67 68 static void nsim_stop_tx_queue(struct net_device *tx_dev, 69 struct net_device *rx_dev, 70 struct nsim_rq *rq, 71 u16 idx) 72 { 73 /* If different queues size, do not stop, since it is not 74 * easy to find which TX queue is mapped here 75 */ 76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) 77 return; 78 79 /* rq is the queue on the receive side */ 80 netif_subqueue_try_stop(tx_dev, idx, 81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), 82 NSIM_RING_SIZE / 2); 83 } 84 85 static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, 86 struct nsim_rq *rq, struct sk_buff *skb) 87 { 88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 89 dev_kfree_skb_any(skb); 90 return NET_RX_DROP; 91 } 92 93 skb_queue_tail(&rq->skb_queue, skb); 94 95 /* Stop the peer TX queue avoiding dropping packets later */ 96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) 97 nsim_stop_tx_queue(tx_dev, rx_dev, rq, 98 skb_get_queue_mapping(skb)); 99 100 return NET_RX_SUCCESS; 101 } 102 103 static int nsim_forward_skb(struct net_device *tx_dev, 104 struct net_device *rx_dev, 105 struct sk_buff *skb, 106 struct nsim_rq *rq, 107 struct skb_ext *psp_ext) 108 { 109 int ret; 110 111 ret = __dev_forward_skb(rx_dev, skb); 112 if (ret) { 113 if (psp_ext) 114 __skb_ext_put(psp_ext); 115 return ret; 116 } 117 118 nsim_psp_handle_ext(skb, psp_ext); 119 120 return nsim_napi_rx(tx_dev, rx_dev, rq, skb); 121 } 122 123 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 124 { 125 struct netdevsim *ns = netdev_priv(dev); 126 struct skb_ext *psp_ext = NULL; 127 struct net_device *peer_dev; 128 unsigned int len = skb->len; 129 struct netdevsim *peer_ns; 130 struct netdev_config *cfg; 131 struct nsim_rq *rq; 132 int rxq; 133 int dr; 134 135 rcu_read_lock(); 136 if (!nsim_ipsec_tx(ns, skb)) 137 goto out_drop_any; 138 139 /* Check if loopback mode is enabled */ 140 if (dev->features & NETIF_F_LOOPBACK) { 141 peer_ns = ns; 142 peer_dev = dev; 143 } else { 144 peer_ns = rcu_dereference(ns->peer); 145 if (!peer_ns) 146 goto out_drop_any; 147 peer_dev = peer_ns->netdev; 148 } 149 150 dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext); 151 if (dr) 152 goto out_drop_free; 153 154 rxq = skb_get_queue_mapping(skb); 155 if (rxq >= peer_dev->num_rx_queues) 156 rxq = rxq % peer_dev->num_rx_queues; 157 rq = peer_ns->rq[rxq]; 158 159 cfg = peer_dev->cfg; 160 if (skb_is_nonlinear(skb) && 161 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 162 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 163 cfg->hds_thresh > len))) 164 skb_linearize(skb); 165 166 skb_tx_timestamp(skb); 167 if (unlikely(nsim_forward_skb(dev, peer_dev, 168 skb, rq, psp_ext) == NET_RX_DROP)) 169 goto out_drop_cnt; 170 171 if (!hrtimer_active(&rq->napi_timer)) 172 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 173 174 rcu_read_unlock(); 175 dev_dstats_tx_add(dev, len); 176 return NETDEV_TX_OK; 177 178 out_drop_any: 179 dr = SKB_DROP_REASON_NOT_SPECIFIED; 180 out_drop_free: 181 kfree_skb_reason(skb, dr); 182 out_drop_cnt: 183 rcu_read_unlock(); 184 dev_dstats_tx_dropped(dev); 185 return NETDEV_TX_OK; 186 } 187 188 static void nsim_set_rx_mode(struct net_device *dev, 189 struct netdev_hw_addr_list *uc, 190 struct netdev_hw_addr_list *mc) 191 { 192 } 193 194 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 195 { 196 struct netdevsim *ns = netdev_priv(dev); 197 198 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 199 new_mtu > NSIM_XDP_MAX_MTU) 200 return -EBUSY; 201 202 WRITE_ONCE(dev->mtu, new_mtu); 203 204 return 0; 205 } 206 207 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 208 { 209 struct netdevsim *ns = netdev_priv(dev); 210 struct nsim_dev *nsim_dev = ns->nsim_dev; 211 212 /* Only refuse multicast addresses, zero address can mean unset/any. */ 213 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 214 return -EINVAL; 215 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 216 217 return 0; 218 } 219 220 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 221 u16 vlan, u8 qos, __be16 vlan_proto) 222 { 223 struct netdevsim *ns = netdev_priv(dev); 224 struct nsim_dev *nsim_dev = ns->nsim_dev; 225 226 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 227 return -EINVAL; 228 229 nsim_dev->vfconfigs[vf].vlan = vlan; 230 nsim_dev->vfconfigs[vf].qos = qos; 231 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 232 233 return 0; 234 } 235 236 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 237 { 238 struct netdevsim *ns = netdev_priv(dev); 239 struct nsim_dev *nsim_dev = ns->nsim_dev; 240 241 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 242 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 243 return -EOPNOTSUPP; 244 } 245 246 if (vf >= nsim_dev_get_vfs(nsim_dev)) 247 return -EINVAL; 248 249 nsim_dev->vfconfigs[vf].min_tx_rate = min; 250 nsim_dev->vfconfigs[vf].max_tx_rate = max; 251 252 return 0; 253 } 254 255 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 256 { 257 struct netdevsim *ns = netdev_priv(dev); 258 struct nsim_dev *nsim_dev = ns->nsim_dev; 259 260 if (vf >= nsim_dev_get_vfs(nsim_dev)) 261 return -EINVAL; 262 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 263 264 return 0; 265 } 266 267 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 268 { 269 struct netdevsim *ns = netdev_priv(dev); 270 struct nsim_dev *nsim_dev = ns->nsim_dev; 271 272 if (vf >= nsim_dev_get_vfs(nsim_dev)) 273 return -EINVAL; 274 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 275 276 return 0; 277 } 278 279 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 280 { 281 struct netdevsim *ns = netdev_priv(dev); 282 struct nsim_dev *nsim_dev = ns->nsim_dev; 283 284 if (vf >= nsim_dev_get_vfs(nsim_dev)) 285 return -EINVAL; 286 nsim_dev->vfconfigs[vf].trusted = val; 287 288 return 0; 289 } 290 291 static int 292 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 293 { 294 struct netdevsim *ns = netdev_priv(dev); 295 struct nsim_dev *nsim_dev = ns->nsim_dev; 296 297 if (vf >= nsim_dev_get_vfs(nsim_dev)) 298 return -EINVAL; 299 300 ivi->vf = vf; 301 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 302 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 303 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 304 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 305 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 306 ivi->qos = nsim_dev->vfconfigs[vf].qos; 307 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 308 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 309 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 310 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 311 312 return 0; 313 } 314 315 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 316 { 317 struct netdevsim *ns = netdev_priv(dev); 318 struct nsim_dev *nsim_dev = ns->nsim_dev; 319 320 if (vf >= nsim_dev_get_vfs(nsim_dev)) 321 return -EINVAL; 322 323 switch (state) { 324 case IFLA_VF_LINK_STATE_AUTO: 325 case IFLA_VF_LINK_STATE_ENABLE: 326 case IFLA_VF_LINK_STATE_DISABLE: 327 break; 328 default: 329 return -EINVAL; 330 } 331 332 nsim_dev->vfconfigs[vf].link_state = state; 333 334 return 0; 335 } 336 337 static int 338 nsim_set_features(struct net_device *dev, netdev_features_t features) 339 { 340 struct netdevsim *ns = netdev_priv(dev); 341 342 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 343 return nsim_bpf_disable_tc(ns); 344 345 return 0; 346 } 347 348 static int nsim_get_iflink(const struct net_device *dev) 349 { 350 struct netdevsim *nsim, *peer; 351 int iflink; 352 353 nsim = netdev_priv(dev); 354 355 rcu_read_lock(); 356 peer = rcu_dereference(nsim->peer); 357 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 358 READ_ONCE(dev->ifindex); 359 rcu_read_unlock(); 360 361 return iflink; 362 } 363 364 static int nsim_rcv(struct nsim_rq *rq, int budget) 365 { 366 struct net_device *dev = rq->napi.dev; 367 struct bpf_prog *xdp_prog; 368 struct netdevsim *ns; 369 struct sk_buff *skb; 370 unsigned int skblen; 371 int i, ret; 372 373 ns = netdev_priv(dev); 374 xdp_prog = READ_ONCE(ns->xdp.prog); 375 376 for (i = 0; i < budget; i++) { 377 if (skb_queue_empty(&rq->skb_queue)) 378 break; 379 380 skb = skb_dequeue(&rq->skb_queue); 381 382 if (xdp_prog) { 383 /* skb might be freed directly by XDP, save the len */ 384 skblen = skb->len; 385 386 if (skb->ip_summed == CHECKSUM_PARTIAL) 387 skb_checksum_help(skb); 388 ret = do_xdp_generic(xdp_prog, &skb); 389 if (ret != XDP_PASS) { 390 dev_dstats_rx_add(dev, skblen); 391 continue; 392 } 393 } 394 395 /* skb might be discard at netif_receive_skb, save the len */ 396 dev_dstats_rx_add(dev, skb->len); 397 napi_gro_receive(&rq->napi, skb); 398 } 399 400 nsim_start_peer_tx_queue(dev, rq); 401 return i; 402 } 403 404 static int nsim_poll(struct napi_struct *napi, int budget) 405 { 406 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 407 int done; 408 409 done = nsim_rcv(rq, budget); 410 if (done < budget) 411 napi_complete_done(napi, done); 412 413 return done; 414 } 415 416 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 417 { 418 struct page_pool_params params = { 419 .order = 0, 420 .pool_size = NSIM_RING_SIZE, 421 .nid = NUMA_NO_NODE, 422 .dev = &napi->dev->dev, 423 .napi = napi, 424 .dma_dir = DMA_BIDIRECTIONAL, 425 .netdev = napi->dev, 426 }; 427 struct page_pool *pool; 428 429 pool = page_pool_create(¶ms); 430 if (IS_ERR(pool)) 431 return PTR_ERR(pool); 432 433 *p = pool; 434 return 0; 435 } 436 437 static int nsim_init_napi(struct netdevsim *ns) 438 { 439 struct net_device *dev = ns->netdev; 440 struct nsim_rq *rq; 441 int err, i; 442 443 for (i = 0; i < dev->num_rx_queues; i++) { 444 rq = ns->rq[i]; 445 446 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 447 } 448 449 for (i = 0; i < dev->num_rx_queues; i++) { 450 rq = ns->rq[i]; 451 452 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 453 if (err) 454 goto err_pp_destroy; 455 } 456 457 return 0; 458 459 err_pp_destroy: 460 while (i--) { 461 page_pool_destroy(ns->rq[i]->page_pool); 462 ns->rq[i]->page_pool = NULL; 463 } 464 465 for (i = 0; i < dev->num_rx_queues; i++) 466 __netif_napi_del_locked(&ns->rq[i]->napi); 467 468 return err; 469 } 470 471 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 472 { 473 struct nsim_rq *rq; 474 475 rq = container_of(timer, struct nsim_rq, napi_timer); 476 napi_schedule(&rq->napi); 477 478 return HRTIMER_NORESTART; 479 } 480 481 static void nsim_rq_timer_init(struct nsim_rq *rq) 482 { 483 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, 484 HRTIMER_MODE_REL); 485 } 486 487 static void nsim_enable_napi(struct netdevsim *ns) 488 { 489 struct net_device *dev = ns->netdev; 490 int i; 491 492 for (i = 0; i < dev->num_rx_queues; i++) { 493 struct nsim_rq *rq = ns->rq[i]; 494 495 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 496 napi_enable_locked(&rq->napi); 497 } 498 } 499 500 static int nsim_open(struct net_device *dev) 501 { 502 struct netdevsim *ns = netdev_priv(dev); 503 struct netdevsim *peer; 504 int err; 505 506 netdev_assert_locked(dev); 507 508 err = nsim_init_napi(ns); 509 if (err) 510 return err; 511 512 nsim_enable_napi(ns); 513 514 peer = rtnl_dereference(ns->peer); 515 if (peer && netif_running(peer->netdev)) { 516 netif_carrier_on(dev); 517 netif_carrier_on(peer->netdev); 518 } 519 520 return 0; 521 } 522 523 static void nsim_del_napi(struct netdevsim *ns) 524 { 525 struct net_device *dev = ns->netdev; 526 int i; 527 528 for (i = 0; i < dev->num_rx_queues; i++) { 529 struct nsim_rq *rq = ns->rq[i]; 530 531 napi_disable_locked(&rq->napi); 532 __netif_napi_del_locked(&rq->napi); 533 } 534 synchronize_net(); 535 536 for (i = 0; i < dev->num_rx_queues; i++) { 537 page_pool_destroy(ns->rq[i]->page_pool); 538 ns->rq[i]->page_pool = NULL; 539 } 540 } 541 542 static int nsim_stop(struct net_device *dev) 543 { 544 struct netdevsim *ns = netdev_priv(dev); 545 struct netdevsim *peer; 546 547 netdev_assert_locked(dev); 548 549 netif_carrier_off(dev); 550 peer = rtnl_dereference(ns->peer); 551 if (peer) 552 netif_carrier_off(peer->netdev); 553 554 nsim_del_napi(ns); 555 556 return 0; 557 } 558 559 static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 560 { 561 struct netdevsim *ns = netdev_priv(dev); 562 563 if (vid >= VLAN_N_VID) 564 return -EINVAL; 565 566 if (proto == htons(ETH_P_8021Q)) 567 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag)); 568 else if (proto == htons(ETH_P_8021AD)) 569 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag)); 570 571 return 0; 572 } 573 574 static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 575 { 576 struct netdevsim *ns = netdev_priv(dev); 577 578 if (vid >= VLAN_N_VID) 579 return -EINVAL; 580 581 if (proto == htons(ETH_P_8021Q)) 582 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag)); 583 else if (proto == htons(ETH_P_8021AD)) 584 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag)); 585 586 return 0; 587 } 588 589 static int nsim_shaper_set(struct net_shaper_binding *binding, 590 const struct net_shaper *shaper, 591 struct netlink_ext_ack *extack) 592 { 593 return 0; 594 } 595 596 static int nsim_shaper_del(struct net_shaper_binding *binding, 597 const struct net_shaper_handle *handle, 598 struct netlink_ext_ack *extack) 599 { 600 return 0; 601 } 602 603 static int nsim_shaper_group(struct net_shaper_binding *binding, 604 int leaves_count, 605 const struct net_shaper *leaves, 606 const struct net_shaper *root, 607 struct netlink_ext_ack *extack) 608 { 609 return 0; 610 } 611 612 static void nsim_shaper_cap(struct net_shaper_binding *binding, 613 enum net_shaper_scope scope, 614 unsigned long *flags) 615 { 616 *flags = ULONG_MAX; 617 } 618 619 static const struct net_shaper_ops nsim_shaper_ops = { 620 .set = nsim_shaper_set, 621 .delete = nsim_shaper_del, 622 .group = nsim_shaper_group, 623 .capabilities = nsim_shaper_cap, 624 }; 625 626 static const struct net_device_ops nsim_netdev_ops = { 627 .ndo_start_xmit = nsim_start_xmit, 628 .ndo_set_rx_mode_async = nsim_set_rx_mode, 629 .ndo_set_mac_address = eth_mac_addr, 630 .ndo_validate_addr = eth_validate_addr, 631 .ndo_change_mtu = nsim_change_mtu, 632 .ndo_set_vf_mac = nsim_set_vf_mac, 633 .ndo_set_vf_vlan = nsim_set_vf_vlan, 634 .ndo_set_vf_rate = nsim_set_vf_rate, 635 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 636 .ndo_set_vf_trust = nsim_set_vf_trust, 637 .ndo_get_vf_config = nsim_get_vf_config, 638 .ndo_set_vf_link_state = nsim_set_vf_link_state, 639 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 640 .ndo_setup_tc = nsim_setup_tc, 641 .ndo_set_features = nsim_set_features, 642 .ndo_get_iflink = nsim_get_iflink, 643 .ndo_bpf = nsim_bpf, 644 .ndo_open = nsim_open, 645 .ndo_stop = nsim_stop, 646 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 647 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 648 .net_shaper_ops = &nsim_shaper_ops, 649 }; 650 651 static const struct net_device_ops nsim_vf_netdev_ops = { 652 .ndo_start_xmit = nsim_start_xmit, 653 .ndo_set_rx_mode_async = nsim_set_rx_mode, 654 .ndo_set_mac_address = eth_mac_addr, 655 .ndo_validate_addr = eth_validate_addr, 656 .ndo_change_mtu = nsim_change_mtu, 657 .ndo_setup_tc = nsim_setup_tc, 658 .ndo_set_features = nsim_set_features, 659 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 660 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 661 }; 662 663 /* We don't have true per-queue stats, yet, so do some random fakery here. 664 * Only report stuff for queue 0. 665 */ 666 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 667 struct netdev_queue_stats_rx *stats) 668 { 669 struct rtnl_link_stats64 rtstats = {}; 670 671 if (!idx) 672 dev_get_stats(dev, &rtstats); 673 674 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 675 stats->bytes = rtstats.rx_bytes; 676 } 677 678 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 679 struct netdev_queue_stats_tx *stats) 680 { 681 struct rtnl_link_stats64 rtstats = {}; 682 683 if (!idx) 684 dev_get_stats(dev, &rtstats); 685 686 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 687 stats->bytes = rtstats.tx_bytes; 688 } 689 690 static void nsim_get_base_stats(struct net_device *dev, 691 struct netdev_queue_stats_rx *rx, 692 struct netdev_queue_stats_tx *tx) 693 { 694 struct rtnl_link_stats64 rtstats = {}; 695 696 dev_get_stats(dev, &rtstats); 697 698 rx->packets = !!rtstats.rx_packets; 699 rx->bytes = 0; 700 tx->packets = !!rtstats.tx_packets; 701 tx->bytes = 0; 702 } 703 704 static const struct netdev_stat_ops nsim_stat_ops = { 705 .get_queue_stats_tx = nsim_get_queue_stats_tx, 706 .get_queue_stats_rx = nsim_get_queue_stats_rx, 707 .get_base_stats = nsim_get_base_stats, 708 }; 709 710 static struct nsim_rq *nsim_queue_alloc(void) 711 { 712 struct nsim_rq *rq; 713 714 rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT); 715 if (!rq) 716 return NULL; 717 718 skb_queue_head_init(&rq->skb_queue); 719 nsim_rq_timer_init(rq); 720 return rq; 721 } 722 723 static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq) 724 { 725 hrtimer_cancel(&rq->napi_timer); 726 727 if (rq->skb_queue.qlen) { 728 local_bh_disable(); 729 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); 730 local_bh_enable(); 731 } 732 733 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 734 kfree(rq); 735 } 736 737 /* Queue reset mode is controlled by ns->rq_reset_mode. 738 * - normal - new NAPI new pool (old NAPI enabled when new added) 739 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 740 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 741 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 742 */ 743 struct nsim_queue_mem { 744 struct nsim_rq *rq; 745 struct page_pool *pp; 746 }; 747 748 static int 749 nsim_queue_mem_alloc(struct net_device *dev, 750 struct netdev_queue_config *qcfg, 751 void *per_queue_mem, int idx) 752 { 753 struct nsim_queue_mem *qmem = per_queue_mem; 754 struct netdevsim *ns = netdev_priv(dev); 755 int err; 756 757 if (ns->rq_reset_mode > 3) 758 return -EINVAL; 759 760 if (ns->rq_reset_mode == 1) { 761 if (!netif_running(ns->netdev)) 762 return -ENETDOWN; 763 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 764 } 765 766 qmem->rq = nsim_queue_alloc(); 767 if (!qmem->rq) 768 return -ENOMEM; 769 770 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 771 if (err) 772 goto err_free; 773 774 if (!ns->rq_reset_mode) 775 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 776 idx); 777 778 return 0; 779 780 err_free: 781 nsim_queue_free(dev, qmem->rq); 782 return err; 783 } 784 785 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 786 { 787 struct nsim_queue_mem *qmem = per_queue_mem; 788 struct netdevsim *ns = netdev_priv(dev); 789 790 page_pool_destroy(qmem->pp); 791 if (qmem->rq) { 792 if (!ns->rq_reset_mode) 793 netif_napi_del_locked(&qmem->rq->napi); 794 page_pool_destroy(qmem->rq->page_pool); 795 nsim_queue_free(dev, qmem->rq); 796 } 797 } 798 799 static int 800 nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, 801 void *per_queue_mem, int idx) 802 { 803 struct nsim_queue_mem *qmem = per_queue_mem; 804 struct netdevsim *ns = netdev_priv(dev); 805 806 netdev_assert_locked(dev); 807 808 if (ns->rq_reset_mode == 1) { 809 ns->rq[idx]->page_pool = qmem->pp; 810 napi_enable_locked(&ns->rq[idx]->napi); 811 return 0; 812 } 813 814 /* netif_napi_add()/_del() should normally be called from alloc/free, 815 * here we want to test various call orders. 816 */ 817 if (ns->rq_reset_mode == 2) { 818 netif_napi_del_locked(&ns->rq[idx]->napi); 819 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 820 idx); 821 } else if (ns->rq_reset_mode == 3) { 822 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 823 idx); 824 netif_napi_del_locked(&ns->rq[idx]->napi); 825 } 826 827 ns->rq[idx] = qmem->rq; 828 napi_enable_locked(&ns->rq[idx]->napi); 829 830 return 0; 831 } 832 833 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 834 { 835 struct nsim_queue_mem *qmem = per_queue_mem; 836 struct netdevsim *ns = netdev_priv(dev); 837 838 netdev_assert_locked(dev); 839 840 napi_disable_locked(&ns->rq[idx]->napi); 841 842 if (ns->rq_reset_mode == 1) { 843 qmem->pp = ns->rq[idx]->page_pool; 844 page_pool_disable_direct_recycling(qmem->pp); 845 } else { 846 qmem->rq = ns->rq[idx]; 847 } 848 849 return 0; 850 } 851 852 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 853 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 854 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 855 .ndo_queue_mem_free = nsim_queue_mem_free, 856 .ndo_queue_start = nsim_queue_start, 857 .ndo_queue_stop = nsim_queue_stop, 858 }; 859 860 static ssize_t 861 nsim_qreset_write(struct file *file, const char __user *data, 862 size_t count, loff_t *ppos) 863 { 864 struct netdevsim *ns = file->private_data; 865 unsigned int queue, mode; 866 char buf[32]; 867 ssize_t ret; 868 869 if (count >= sizeof(buf)) 870 return -EINVAL; 871 if (copy_from_user(buf, data, count)) 872 return -EFAULT; 873 buf[count] = '\0'; 874 875 ret = sscanf(buf, "%u %u", &queue, &mode); 876 if (ret != 2) 877 return -EINVAL; 878 879 netdev_lock(ns->netdev); 880 if (queue >= ns->netdev->real_num_rx_queues) { 881 ret = -EINVAL; 882 goto exit_unlock; 883 } 884 885 ns->rq_reset_mode = mode; 886 ret = netdev_rx_queue_restart(ns->netdev, queue); 887 ns->rq_reset_mode = 0; 888 if (ret) 889 goto exit_unlock; 890 891 ret = count; 892 exit_unlock: 893 netdev_unlock(ns->netdev); 894 return ret; 895 } 896 897 static const struct file_operations nsim_qreset_fops = { 898 .open = simple_open, 899 .write = nsim_qreset_write, 900 .owner = THIS_MODULE, 901 }; 902 903 static ssize_t 904 nsim_pp_hold_read(struct file *file, char __user *data, 905 size_t count, loff_t *ppos) 906 { 907 struct netdevsim *ns = file->private_data; 908 char buf[3] = "n\n"; 909 910 if (ns->page) 911 buf[0] = 'y'; 912 913 return simple_read_from_buffer(data, count, ppos, buf, 2); 914 } 915 916 static ssize_t 917 nsim_pp_hold_write(struct file *file, const char __user *data, 918 size_t count, loff_t *ppos) 919 { 920 struct netdevsim *ns = file->private_data; 921 ssize_t ret; 922 bool val; 923 924 ret = kstrtobool_from_user(data, count, &val); 925 if (ret) 926 return ret; 927 928 rtnl_lock(); 929 ret = count; 930 if (val == !!ns->page) 931 goto exit; 932 933 if (!netif_running(ns->netdev) && val) { 934 ret = -ENETDOWN; 935 } else if (val) { 936 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 937 if (!ns->page) 938 ret = -ENOMEM; 939 } else { 940 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 941 ns->page, false); 942 ns->page = NULL; 943 } 944 945 exit: 946 rtnl_unlock(); 947 return ret; 948 } 949 950 static const struct file_operations nsim_pp_hold_fops = { 951 .open = simple_open, 952 .read = nsim_pp_hold_read, 953 .write = nsim_pp_hold_write, 954 .llseek = generic_file_llseek, 955 .owner = THIS_MODULE, 956 }; 957 958 static int nsim_vlan_show(struct seq_file *s, void *data) 959 { 960 struct netdevsim *ns = s->private; 961 int vid; 962 963 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 964 seq_printf(s, "ctag %d\n", vid); 965 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 966 seq_printf(s, "stag %d\n", vid); 967 968 return 0; 969 } 970 DEFINE_SHOW_ATTRIBUTE(nsim_vlan); 971 972 static void nsim_setup(struct net_device *dev) 973 { 974 ether_setup(dev); 975 eth_hw_addr_random(dev); 976 977 dev->flags &= ~IFF_MULTICAST; 978 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 979 dev->features |= NETIF_F_HIGHDMA | 980 NETIF_F_SG | 981 NETIF_F_FRAGLIST | 982 NETIF_F_HW_CSUM | 983 NETIF_F_LRO | 984 NETIF_F_TSO | 985 NETIF_F_HW_VLAN_CTAG_FILTER | 986 NETIF_F_HW_VLAN_STAG_FILTER; 987 dev->hw_features |= NETIF_F_HW_TC | 988 NETIF_F_SG | 989 NETIF_F_FRAGLIST | 990 NETIF_F_HW_CSUM | 991 NETIF_F_LRO | 992 NETIF_F_TSO | 993 NETIF_F_LOOPBACK | 994 NETIF_F_HW_VLAN_CTAG_FILTER | 995 NETIF_F_HW_VLAN_STAG_FILTER; 996 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; 997 dev->max_mtu = ETH_MAX_MTU; 998 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD; 999 } 1000 1001 static int nsim_queue_init(struct netdevsim *ns) 1002 { 1003 struct net_device *dev = ns->netdev; 1004 int i; 1005 1006 ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT); 1007 if (!ns->rq) 1008 return -ENOMEM; 1009 1010 for (i = 0; i < dev->num_rx_queues; i++) { 1011 ns->rq[i] = nsim_queue_alloc(); 1012 if (!ns->rq[i]) 1013 goto err_free_prev; 1014 } 1015 1016 return 0; 1017 1018 err_free_prev: 1019 while (i--) 1020 kfree(ns->rq[i]); 1021 kfree(ns->rq); 1022 return -ENOMEM; 1023 } 1024 1025 static void nsim_queue_uninit(struct netdevsim *ns) 1026 { 1027 struct net_device *dev = ns->netdev; 1028 int i; 1029 1030 for (i = 0; i < dev->num_rx_queues; i++) 1031 nsim_queue_free(dev, ns->rq[i]); 1032 1033 kfree(ns->rq); 1034 ns->rq = NULL; 1035 } 1036 1037 static int nsim_init_netdevsim(struct netdevsim *ns) 1038 { 1039 struct netdevsim *peer; 1040 struct mock_phc *phc; 1041 int err; 1042 1043 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 1044 if (IS_ERR(phc)) 1045 return PTR_ERR(phc); 1046 1047 ns->phc = phc; 1048 ns->netdev->netdev_ops = &nsim_netdev_ops; 1049 ns->netdev->stat_ops = &nsim_stat_ops; 1050 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 1051 netdev_lockdep_set_classes(ns->netdev); 1052 1053 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 1054 if (err) 1055 goto err_phc_destroy; 1056 1057 rtnl_lock(); 1058 err = nsim_queue_init(ns); 1059 if (err) 1060 goto err_utn_destroy; 1061 1062 err = nsim_bpf_init(ns); 1063 if (err) 1064 goto err_rq_destroy; 1065 1066 nsim_macsec_init(ns); 1067 nsim_ipsec_init(ns); 1068 1069 err = register_netdevice(ns->netdev); 1070 if (err) 1071 goto err_ipsec_teardown; 1072 rtnl_unlock(); 1073 1074 err = nsim_psp_init(ns); 1075 if (err) 1076 goto err_unregister_netdev; 1077 1078 if (IS_ENABLED(CONFIG_DEBUG_NET)) { 1079 ns->nb.notifier_call = netdev_debug_event; 1080 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1081 &ns->nn)) 1082 ns->nb.notifier_call = NULL; 1083 } 1084 1085 return 0; 1086 1087 err_unregister_netdev: 1088 rtnl_lock(); 1089 peer = rtnl_dereference(ns->peer); 1090 if (peer) 1091 RCU_INIT_POINTER(peer->peer, NULL); 1092 RCU_INIT_POINTER(ns->peer, NULL); 1093 unregister_netdevice(ns->netdev); 1094 err_ipsec_teardown: 1095 nsim_ipsec_teardown(ns); 1096 nsim_macsec_teardown(ns); 1097 nsim_bpf_uninit(ns); 1098 err_rq_destroy: 1099 nsim_queue_uninit(ns); 1100 err_utn_destroy: 1101 rtnl_unlock(); 1102 nsim_udp_tunnels_info_destroy(ns->netdev); 1103 err_phc_destroy: 1104 mock_phc_destroy(ns->phc); 1105 return err; 1106 } 1107 1108 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 1109 { 1110 int err; 1111 1112 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 1113 rtnl_lock(); 1114 err = register_netdevice(ns->netdev); 1115 rtnl_unlock(); 1116 return err; 1117 } 1118 1119 static void nsim_exit_netdevsim(struct netdevsim *ns) 1120 { 1121 nsim_udp_tunnels_info_destroy(ns->netdev); 1122 mock_phc_destroy(ns->phc); 1123 } 1124 1125 struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, 1126 struct nsim_dev_port *nsim_dev_port, 1127 u8 perm_addr[ETH_ALEN]) 1128 { 1129 struct net_device *dev; 1130 struct netdevsim *ns; 1131 int err; 1132 1133 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1134 nsim_dev->nsim_bus_dev->num_queues); 1135 if (!dev) 1136 return ERR_PTR(-ENOMEM); 1137 1138 if (perm_addr) 1139 memcpy(dev->perm_addr, perm_addr, ETH_ALEN); 1140 1141 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1142 ns = netdev_priv(dev); 1143 ns->netdev = dev; 1144 ns->nsim_dev = nsim_dev; 1145 ns->nsim_dev_port = nsim_dev_port; 1146 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1147 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1148 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1149 nsim_ethtool_init(ns); 1150 if (nsim_dev_port_is_pf(nsim_dev_port)) 1151 err = nsim_init_netdevsim(ns); 1152 else 1153 err = nsim_init_netdevsim_vf(ns); 1154 if (err) 1155 goto err_free_netdev; 1156 1157 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1158 ns, &nsim_pp_hold_fops); 1159 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1160 nsim_dev_port->ddir, ns, 1161 &nsim_qreset_fops); 1162 ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir, 1163 ns, &nsim_vlan_fops); 1164 return ns; 1165 1166 err_free_netdev: 1167 free_netdev(dev); 1168 return ERR_PTR(err); 1169 } 1170 1171 void nsim_destroy(struct netdevsim *ns) 1172 { 1173 struct net_device *dev = ns->netdev; 1174 struct netdevsim *peer; 1175 u16 vid; 1176 1177 debugfs_remove(ns->vlan_dfs); 1178 debugfs_remove(ns->qr_dfs); 1179 debugfs_remove(ns->pp_dfs); 1180 1181 if (ns->nb.notifier_call) 1182 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1183 &ns->nn); 1184 1185 nsim_psp_uninit(ns); 1186 1187 rtnl_lock(); 1188 peer = rtnl_dereference(ns->peer); 1189 if (peer) 1190 RCU_INIT_POINTER(peer->peer, NULL); 1191 RCU_INIT_POINTER(ns->peer, NULL); 1192 unregister_netdevice(dev); 1193 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1194 nsim_macsec_teardown(ns); 1195 nsim_ipsec_teardown(ns); 1196 nsim_bpf_uninit(ns); 1197 nsim_queue_uninit(ns); 1198 } 1199 rtnl_unlock(); 1200 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1201 nsim_exit_netdevsim(ns); 1202 1203 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 1204 WARN_ON_ONCE(1); 1205 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 1206 WARN_ON_ONCE(1); 1207 1208 /* Put this intentionally late to exercise the orphaning path */ 1209 if (ns->page) { 1210 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 1211 ns->page, false); 1212 ns->page = NULL; 1213 } 1214 1215 free_netdev(dev); 1216 } 1217 1218 bool netdev_is_nsim(struct net_device *dev) 1219 { 1220 return dev->netdev_ops == &nsim_netdev_ops; 1221 } 1222 1223 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1224 struct netlink_ext_ack *extack) 1225 { 1226 NL_SET_ERR_MSG_MOD(extack, 1227 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1228 return -EOPNOTSUPP; 1229 } 1230 1231 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1232 .kind = DRV_NAME, 1233 .validate = nsim_validate, 1234 }; 1235 1236 static int __init nsim_module_init(void) 1237 { 1238 int err; 1239 1240 err = nsim_dev_init(); 1241 if (err) 1242 return err; 1243 1244 err = nsim_bus_init(); 1245 if (err) 1246 goto err_dev_exit; 1247 1248 err = rtnl_link_register(&nsim_link_ops); 1249 if (err) 1250 goto err_bus_exit; 1251 1252 return 0; 1253 1254 err_bus_exit: 1255 nsim_bus_exit(); 1256 err_dev_exit: 1257 nsim_dev_exit(); 1258 return err; 1259 } 1260 1261 static void __exit nsim_module_exit(void) 1262 { 1263 rtnl_link_unregister(&nsim_link_ops); 1264 nsim_bus_exit(); 1265 nsim_dev_exit(); 1266 } 1267 1268 module_init(nsim_module_init); 1269 module_exit(nsim_module_exit); 1270 MODULE_LICENSE("GPL"); 1271 MODULE_DESCRIPTION("Simulated networking device for testing"); 1272 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1273