1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/debugfs.h> 17 #include <linux/etherdevice.h> 18 #include <linux/ethtool_netlink.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/netdevice.h> 22 #include <linux/slab.h> 23 #include <net/netdev_queues.h> 24 #include <net/netdev_rx_queue.h> 25 #include <net/page_pool/helpers.h> 26 #include <net/netlink.h> 27 #include <net/net_shaper.h> 28 #include <net/netdev_lock.h> 29 #include <net/pkt_cls.h> 30 #include <net/rtnetlink.h> 31 #include <net/udp_tunnel.h> 32 #include <net/busy_poll.h> 33 34 #include "netdevsim.h" 35 36 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 37 38 #define NSIM_RING_SIZE 256 39 40 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq) 41 { 42 struct netdevsim *ns = netdev_priv(dev); 43 struct net_device *peer_dev; 44 struct netdevsim *peer_ns; 45 struct netdev_queue *txq; 46 u16 idx; 47 48 idx = rq->napi.index; 49 rcu_read_lock(); 50 peer_ns = rcu_dereference(ns->peer); 51 if (!peer_ns) 52 goto out; 53 54 /* TX device */ 55 peer_dev = peer_ns->netdev; 56 if (dev->real_num_tx_queues != peer_dev->num_rx_queues) 57 goto out; 58 59 txq = netdev_get_tx_queue(peer_dev, idx); 60 if (!netif_tx_queue_stopped(txq)) 61 goto out; 62 63 netif_tx_wake_queue(txq); 64 out: 65 rcu_read_unlock(); 66 } 67 68 static void nsim_stop_tx_queue(struct net_device *tx_dev, 69 struct net_device *rx_dev, 70 struct nsim_rq *rq, 71 u16 idx) 72 { 73 /* If different queues size, do not stop, since it is not 74 * easy to find which TX queue is mapped here 75 */ 76 if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues) 77 return; 78 79 /* rq is the queue on the receive side */ 80 netif_subqueue_try_stop(tx_dev, idx, 81 NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue), 82 NSIM_RING_SIZE / 2); 83 } 84 85 static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev, 86 struct nsim_rq *rq, struct sk_buff *skb) 87 { 88 if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) { 89 dev_kfree_skb_any(skb); 90 return NET_RX_DROP; 91 } 92 93 skb_queue_tail(&rq->skb_queue, skb); 94 95 /* Stop the peer TX queue avoiding dropping packets later */ 96 if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE) 97 nsim_stop_tx_queue(tx_dev, rx_dev, rq, 98 skb_get_queue_mapping(skb)); 99 100 return NET_RX_SUCCESS; 101 } 102 103 static int nsim_forward_skb(struct net_device *tx_dev, 104 struct net_device *rx_dev, 105 struct sk_buff *skb, 106 struct nsim_rq *rq, 107 struct skb_ext *psp_ext) 108 { 109 int ret; 110 111 ret = __dev_forward_skb(rx_dev, skb); 112 if (ret) { 113 if (psp_ext) 114 __skb_ext_put(psp_ext); 115 return ret; 116 } 117 118 nsim_psp_handle_ext(skb, psp_ext); 119 120 return nsim_napi_rx(tx_dev, rx_dev, rq, skb); 121 } 122 123 static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev) 124 { 125 struct netdevsim *ns = netdev_priv(dev); 126 struct skb_ext *psp_ext = NULL; 127 struct net_device *peer_dev; 128 unsigned int len = skb->len; 129 struct netdevsim *peer_ns; 130 struct netdev_config *cfg; 131 struct nsim_rq *rq; 132 int rxq; 133 int dr; 134 135 rcu_read_lock(); 136 if (!nsim_ipsec_tx(ns, skb)) 137 goto out_drop_any; 138 139 /* Check if loopback mode is enabled */ 140 if (dev->features & NETIF_F_LOOPBACK) { 141 peer_ns = ns; 142 peer_dev = dev; 143 } else { 144 peer_ns = rcu_dereference(ns->peer); 145 if (!peer_ns) 146 goto out_drop_any; 147 peer_dev = peer_ns->netdev; 148 } 149 150 dr = nsim_do_psp(skb, ns, peer_ns, &psp_ext); 151 if (dr) 152 goto out_drop_free; 153 154 rxq = skb_get_queue_mapping(skb); 155 if (rxq >= peer_dev->num_rx_queues) 156 rxq = rxq % peer_dev->num_rx_queues; 157 rq = peer_ns->rq[rxq]; 158 159 cfg = peer_dev->cfg; 160 if (skb_is_nonlinear(skb) && 161 (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED || 162 (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 163 cfg->hds_thresh > len))) 164 skb_linearize(skb); 165 166 skb_tx_timestamp(skb); 167 if (unlikely(nsim_forward_skb(dev, peer_dev, 168 skb, rq, psp_ext) == NET_RX_DROP)) 169 goto out_drop_cnt; 170 171 if (!hrtimer_active(&rq->napi_timer)) 172 hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 173 174 rcu_read_unlock(); 175 dev_dstats_tx_add(dev, len); 176 return NETDEV_TX_OK; 177 178 out_drop_any: 179 dr = SKB_DROP_REASON_NOT_SPECIFIED; 180 out_drop_free: 181 kfree_skb_reason(skb, dr); 182 out_drop_cnt: 183 rcu_read_unlock(); 184 dev_dstats_tx_dropped(dev); 185 return NETDEV_TX_OK; 186 } 187 188 static netdev_tx_t nsim_start_xmit_vf(struct sk_buff *skb, 189 struct net_device *dev) 190 { 191 dev_dstats_tx_dropped(dev); 192 kfree_skb(skb); 193 return NETDEV_TX_OK; 194 } 195 196 static int nsim_set_rx_mode(struct net_device *dev, 197 struct netdev_hw_addr_list *uc, 198 struct netdev_hw_addr_list *mc) 199 { 200 return 0; 201 } 202 203 static int nsim_change_mtu(struct net_device *dev, int new_mtu) 204 { 205 struct netdevsim *ns = netdev_priv(dev); 206 207 if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags && 208 new_mtu > NSIM_XDP_MAX_MTU) 209 return -EBUSY; 210 211 WRITE_ONCE(dev->mtu, new_mtu); 212 213 return 0; 214 } 215 216 static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 217 { 218 struct netdevsim *ns = netdev_priv(dev); 219 struct nsim_dev *nsim_dev = ns->nsim_dev; 220 221 /* Only refuse multicast addresses, zero address can mean unset/any. */ 222 if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac)) 223 return -EINVAL; 224 memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 225 226 return 0; 227 } 228 229 static int nsim_set_vf_vlan(struct net_device *dev, int vf, 230 u16 vlan, u8 qos, __be16 vlan_proto) 231 { 232 struct netdevsim *ns = netdev_priv(dev); 233 struct nsim_dev *nsim_dev = ns->nsim_dev; 234 235 if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7) 236 return -EINVAL; 237 238 nsim_dev->vfconfigs[vf].vlan = vlan; 239 nsim_dev->vfconfigs[vf].qos = qos; 240 nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 241 242 return 0; 243 } 244 245 static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max) 246 { 247 struct netdevsim *ns = netdev_priv(dev); 248 struct nsim_dev *nsim_dev = ns->nsim_dev; 249 250 if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) { 251 pr_err("Not supported in switchdev mode. Please use devlink API.\n"); 252 return -EOPNOTSUPP; 253 } 254 255 if (vf >= nsim_dev_get_vfs(nsim_dev)) 256 return -EINVAL; 257 258 nsim_dev->vfconfigs[vf].min_tx_rate = min; 259 nsim_dev->vfconfigs[vf].max_tx_rate = max; 260 261 return 0; 262 } 263 264 static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val) 265 { 266 struct netdevsim *ns = netdev_priv(dev); 267 struct nsim_dev *nsim_dev = ns->nsim_dev; 268 269 if (vf >= nsim_dev_get_vfs(nsim_dev)) 270 return -EINVAL; 271 nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 272 273 return 0; 274 } 275 276 static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val) 277 { 278 struct netdevsim *ns = netdev_priv(dev); 279 struct nsim_dev *nsim_dev = ns->nsim_dev; 280 281 if (vf >= nsim_dev_get_vfs(nsim_dev)) 282 return -EINVAL; 283 nsim_dev->vfconfigs[vf].rss_query_enabled = val; 284 285 return 0; 286 } 287 288 static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val) 289 { 290 struct netdevsim *ns = netdev_priv(dev); 291 struct nsim_dev *nsim_dev = ns->nsim_dev; 292 293 if (vf >= nsim_dev_get_vfs(nsim_dev)) 294 return -EINVAL; 295 nsim_dev->vfconfigs[vf].trusted = val; 296 297 return 0; 298 } 299 300 static int 301 nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi) 302 { 303 struct netdevsim *ns = netdev_priv(dev); 304 struct nsim_dev *nsim_dev = ns->nsim_dev; 305 306 if (vf >= nsim_dev_get_vfs(nsim_dev)) 307 return -EINVAL; 308 309 ivi->vf = vf; 310 ivi->linkstate = nsim_dev->vfconfigs[vf].link_state; 311 ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate; 312 ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate; 313 ivi->vlan = nsim_dev->vfconfigs[vf].vlan; 314 ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto; 315 ivi->qos = nsim_dev->vfconfigs[vf].qos; 316 memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN); 317 ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled; 318 ivi->trusted = nsim_dev->vfconfigs[vf].trusted; 319 ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 320 321 return 0; 322 } 323 324 static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state) 325 { 326 struct netdevsim *ns = netdev_priv(dev); 327 struct nsim_dev *nsim_dev = ns->nsim_dev; 328 329 if (vf >= nsim_dev_get_vfs(nsim_dev)) 330 return -EINVAL; 331 332 switch (state) { 333 case IFLA_VF_LINK_STATE_AUTO: 334 case IFLA_VF_LINK_STATE_ENABLE: 335 case IFLA_VF_LINK_STATE_DISABLE: 336 break; 337 default: 338 return -EINVAL; 339 } 340 341 nsim_dev->vfconfigs[vf].link_state = state; 342 343 return 0; 344 } 345 346 static int 347 nsim_set_features(struct net_device *dev, netdev_features_t features) 348 { 349 struct netdevsim *ns = netdev_priv(dev); 350 351 if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC)) 352 return nsim_bpf_disable_tc(ns); 353 354 return 0; 355 } 356 357 static int nsim_get_iflink(const struct net_device *dev) 358 { 359 struct netdevsim *nsim, *peer; 360 int iflink; 361 362 nsim = netdev_priv(dev); 363 364 rcu_read_lock(); 365 peer = rcu_dereference(nsim->peer); 366 iflink = peer ? READ_ONCE(peer->netdev->ifindex) : 367 READ_ONCE(dev->ifindex); 368 rcu_read_unlock(); 369 370 return iflink; 371 } 372 373 static int nsim_rcv(struct nsim_rq *rq, int budget) 374 { 375 struct net_device *dev = rq->napi.dev; 376 struct bpf_prog *xdp_prog; 377 struct netdevsim *ns; 378 struct sk_buff *skb; 379 unsigned int skblen; 380 int i, ret; 381 382 ns = netdev_priv(dev); 383 xdp_prog = READ_ONCE(ns->xdp.prog); 384 385 for (i = 0; i < budget; i++) { 386 if (skb_queue_empty(&rq->skb_queue)) 387 break; 388 389 skb = skb_dequeue(&rq->skb_queue); 390 391 if (xdp_prog) { 392 /* skb might be freed directly by XDP, save the len */ 393 skblen = skb->len; 394 395 if (skb->ip_summed == CHECKSUM_PARTIAL) 396 skb_checksum_help(skb); 397 ret = do_xdp_generic(xdp_prog, &skb); 398 if (ret != XDP_PASS) { 399 dev_dstats_rx_add(dev, skblen); 400 continue; 401 } 402 } 403 404 /* skb might be discard at netif_receive_skb, save the len */ 405 dev_dstats_rx_add(dev, skb->len); 406 napi_gro_receive(&rq->napi, skb); 407 } 408 409 nsim_start_peer_tx_queue(dev, rq); 410 return i; 411 } 412 413 static int nsim_poll(struct napi_struct *napi, int budget) 414 { 415 struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi); 416 int done; 417 418 done = nsim_rcv(rq, budget); 419 if (done < budget) 420 napi_complete_done(napi, done); 421 422 return done; 423 } 424 425 static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi) 426 { 427 struct page_pool_params params = { 428 .order = 0, 429 .pool_size = NSIM_RING_SIZE, 430 .nid = NUMA_NO_NODE, 431 .dev = &napi->dev->dev, 432 .napi = napi, 433 .dma_dir = DMA_BIDIRECTIONAL, 434 .netdev = napi->dev, 435 }; 436 struct page_pool *pool; 437 438 pool = page_pool_create(¶ms); 439 if (IS_ERR(pool)) 440 return PTR_ERR(pool); 441 442 *p = pool; 443 return 0; 444 } 445 446 static int nsim_init_napi(struct netdevsim *ns) 447 { 448 struct net_device *dev = ns->netdev; 449 struct nsim_rq *rq; 450 int err, i; 451 452 for (i = 0; i < dev->num_rx_queues; i++) { 453 rq = ns->rq[i]; 454 455 netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i); 456 } 457 458 for (i = 0; i < dev->num_rx_queues; i++) { 459 rq = ns->rq[i]; 460 461 err = nsim_create_page_pool(&rq->page_pool, &rq->napi); 462 if (err) 463 goto err_pp_destroy; 464 } 465 466 return 0; 467 468 err_pp_destroy: 469 while (i--) { 470 page_pool_destroy(ns->rq[i]->page_pool); 471 ns->rq[i]->page_pool = NULL; 472 } 473 474 for (i = 0; i < dev->num_rx_queues; i++) 475 __netif_napi_del_locked(&ns->rq[i]->napi); 476 477 return err; 478 } 479 480 static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer) 481 { 482 struct nsim_rq *rq; 483 484 rq = container_of(timer, struct nsim_rq, napi_timer); 485 napi_schedule(&rq->napi); 486 487 return HRTIMER_NORESTART; 488 } 489 490 static void nsim_rq_timer_init(struct nsim_rq *rq) 491 { 492 hrtimer_setup(&rq->napi_timer, nsim_napi_schedule, CLOCK_MONOTONIC, 493 HRTIMER_MODE_REL); 494 } 495 496 static void nsim_enable_napi(struct netdevsim *ns) 497 { 498 struct net_device *dev = ns->netdev; 499 int i; 500 501 for (i = 0; i < dev->num_rx_queues; i++) { 502 struct nsim_rq *rq = ns->rq[i]; 503 504 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, &rq->napi); 505 napi_enable_locked(&rq->napi); 506 } 507 } 508 509 static int nsim_open(struct net_device *dev) 510 { 511 struct netdevsim *ns = netdev_priv(dev); 512 struct netdevsim *peer; 513 int err; 514 515 netdev_assert_locked(dev); 516 517 err = nsim_init_napi(ns); 518 if (err) 519 return err; 520 521 nsim_enable_napi(ns); 522 523 peer = rtnl_dereference(ns->peer); 524 if (peer && netif_running(peer->netdev)) { 525 netif_carrier_on(dev); 526 netif_carrier_on(peer->netdev); 527 } 528 529 return 0; 530 } 531 532 static void nsim_del_napi(struct netdevsim *ns) 533 { 534 struct net_device *dev = ns->netdev; 535 int i; 536 537 for (i = 0; i < dev->num_rx_queues; i++) { 538 struct nsim_rq *rq = ns->rq[i]; 539 540 netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL); 541 napi_disable_locked(&rq->napi); 542 __netif_napi_del_locked(&rq->napi); 543 } 544 synchronize_net(); 545 546 for (i = 0; i < dev->num_rx_queues; i++) { 547 page_pool_destroy(ns->rq[i]->page_pool); 548 ns->rq[i]->page_pool = NULL; 549 } 550 } 551 552 static int nsim_stop(struct net_device *dev) 553 { 554 struct netdevsim *ns = netdev_priv(dev); 555 struct netdevsim *peer; 556 557 netdev_assert_locked(dev); 558 559 netif_carrier_off(dev); 560 peer = rtnl_dereference(ns->peer); 561 if (peer) 562 netif_carrier_off(peer->netdev); 563 564 nsim_del_napi(ns); 565 566 return 0; 567 } 568 569 static int nsim_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 570 { 571 struct netdevsim *ns = netdev_priv(dev); 572 573 if (vid >= VLAN_N_VID) 574 return -EINVAL; 575 576 if (proto == htons(ETH_P_8021Q)) 577 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.ctag)); 578 else if (proto == htons(ETH_P_8021AD)) 579 WARN_ON_ONCE(test_and_set_bit(vid, ns->vlan.stag)); 580 581 return 0; 582 } 583 584 static int nsim_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 585 { 586 struct netdevsim *ns = netdev_priv(dev); 587 588 if (vid >= VLAN_N_VID) 589 return -EINVAL; 590 591 if (proto == htons(ETH_P_8021Q)) 592 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.ctag)); 593 else if (proto == htons(ETH_P_8021AD)) 594 WARN_ON_ONCE(!test_and_clear_bit(vid, ns->vlan.stag)); 595 596 return 0; 597 } 598 599 static int nsim_shaper_set(struct net_shaper_binding *binding, 600 const struct net_shaper *shaper, 601 struct netlink_ext_ack *extack) 602 { 603 return 0; 604 } 605 606 static int nsim_shaper_del(struct net_shaper_binding *binding, 607 const struct net_shaper_handle *handle, 608 struct netlink_ext_ack *extack) 609 { 610 return 0; 611 } 612 613 static int nsim_shaper_group(struct net_shaper_binding *binding, 614 int leaves_count, 615 const struct net_shaper *leaves, 616 const struct net_shaper *root, 617 struct netlink_ext_ack *extack) 618 { 619 return 0; 620 } 621 622 static void nsim_shaper_cap(struct net_shaper_binding *binding, 623 enum net_shaper_scope scope, 624 unsigned long *flags) 625 { 626 *flags = ULONG_MAX; 627 } 628 629 static const struct net_shaper_ops nsim_shaper_ops = { 630 .set = nsim_shaper_set, 631 .delete = nsim_shaper_del, 632 .group = nsim_shaper_group, 633 .capabilities = nsim_shaper_cap, 634 }; 635 636 static const struct net_device_ops nsim_netdev_ops = { 637 .ndo_start_xmit = nsim_start_xmit, 638 .ndo_set_rx_mode_async = nsim_set_rx_mode, 639 .ndo_set_mac_address = eth_mac_addr, 640 .ndo_validate_addr = eth_validate_addr, 641 .ndo_change_mtu = nsim_change_mtu, 642 .ndo_set_vf_mac = nsim_set_vf_mac, 643 .ndo_set_vf_vlan = nsim_set_vf_vlan, 644 .ndo_set_vf_rate = nsim_set_vf_rate, 645 .ndo_set_vf_spoofchk = nsim_set_vf_spoofchk, 646 .ndo_set_vf_trust = nsim_set_vf_trust, 647 .ndo_get_vf_config = nsim_get_vf_config, 648 .ndo_set_vf_link_state = nsim_set_vf_link_state, 649 .ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en, 650 .ndo_setup_tc = nsim_setup_tc, 651 .ndo_set_features = nsim_set_features, 652 .ndo_get_iflink = nsim_get_iflink, 653 .ndo_bpf = nsim_bpf, 654 .ndo_open = nsim_open, 655 .ndo_stop = nsim_stop, 656 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 657 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 658 .net_shaper_ops = &nsim_shaper_ops, 659 }; 660 661 static const struct net_device_ops nsim_vf_netdev_ops = { 662 .ndo_start_xmit = nsim_start_xmit_vf, 663 .ndo_set_rx_mode_async = nsim_set_rx_mode, 664 .ndo_set_mac_address = eth_mac_addr, 665 .ndo_validate_addr = eth_validate_addr, 666 .ndo_change_mtu = nsim_change_mtu, 667 .ndo_setup_tc = nsim_setup_tc, 668 .ndo_set_features = nsim_set_features, 669 .ndo_vlan_rx_add_vid = nsim_vlan_rx_add_vid, 670 .ndo_vlan_rx_kill_vid = nsim_vlan_rx_kill_vid, 671 }; 672 673 /* We don't have true per-queue stats, yet, so do some random fakery here. 674 * Only report stuff for queue 0. 675 */ 676 static void nsim_get_queue_stats_rx(struct net_device *dev, int idx, 677 struct netdev_queue_stats_rx *stats) 678 { 679 struct rtnl_link_stats64 rtstats = {}; 680 681 if (!idx) 682 dev_get_stats(dev, &rtstats); 683 684 stats->packets = rtstats.rx_packets - !!rtstats.rx_packets; 685 stats->bytes = rtstats.rx_bytes; 686 } 687 688 static void nsim_get_queue_stats_tx(struct net_device *dev, int idx, 689 struct netdev_queue_stats_tx *stats) 690 { 691 struct rtnl_link_stats64 rtstats = {}; 692 693 if (!idx) 694 dev_get_stats(dev, &rtstats); 695 696 stats->packets = rtstats.tx_packets - !!rtstats.tx_packets; 697 stats->bytes = rtstats.tx_bytes; 698 } 699 700 static void nsim_get_base_stats(struct net_device *dev, 701 struct netdev_queue_stats_rx *rx, 702 struct netdev_queue_stats_tx *tx) 703 { 704 struct rtnl_link_stats64 rtstats = {}; 705 706 dev_get_stats(dev, &rtstats); 707 708 rx->packets = !!rtstats.rx_packets; 709 rx->bytes = 0; 710 tx->packets = !!rtstats.tx_packets; 711 tx->bytes = 0; 712 } 713 714 static const struct netdev_stat_ops nsim_stat_ops = { 715 .get_queue_stats_tx = nsim_get_queue_stats_tx, 716 .get_queue_stats_rx = nsim_get_queue_stats_rx, 717 .get_base_stats = nsim_get_base_stats, 718 }; 719 720 static struct nsim_rq *nsim_queue_alloc(void) 721 { 722 struct nsim_rq *rq; 723 724 rq = kzalloc_obj(*rq, GFP_KERNEL_ACCOUNT); 725 if (!rq) 726 return NULL; 727 728 skb_queue_head_init(&rq->skb_queue); 729 nsim_rq_timer_init(rq); 730 return rq; 731 } 732 733 static void nsim_queue_free(struct net_device *dev, struct nsim_rq *rq) 734 { 735 hrtimer_cancel(&rq->napi_timer); 736 737 if (rq->skb_queue.qlen) { 738 local_bh_disable(); 739 dev_dstats_rx_dropped_add(dev, rq->skb_queue.qlen); 740 local_bh_enable(); 741 } 742 743 skb_queue_purge_reason(&rq->skb_queue, SKB_DROP_REASON_QUEUE_PURGE); 744 kfree(rq); 745 } 746 747 /* Queue reset mode is controlled by ns->rq_reset_mode. 748 * - normal - new NAPI new pool (old NAPI enabled when new added) 749 * - mode 1 - allocate new pool (NAPI is only disabled / enabled) 750 * - mode 2 - new NAPI new pool (old NAPI removed before new added) 751 * - mode 3 - new NAPI new pool (old NAPI disabled when new added) 752 */ 753 struct nsim_queue_mem { 754 struct nsim_rq *rq; 755 struct page_pool *pp; 756 }; 757 758 static int 759 nsim_queue_mem_alloc(struct net_device *dev, 760 struct netdev_queue_config *qcfg, 761 void *per_queue_mem, int idx) 762 { 763 struct nsim_queue_mem *qmem = per_queue_mem; 764 struct netdevsim *ns = netdev_priv(dev); 765 int err; 766 767 if (ns->rq_reset_mode > 3) 768 return -EINVAL; 769 770 if (ns->rq_reset_mode == 1) { 771 if (!netif_running(ns->netdev)) 772 return -ENETDOWN; 773 return nsim_create_page_pool(&qmem->pp, &ns->rq[idx]->napi); 774 } 775 776 qmem->rq = nsim_queue_alloc(); 777 if (!qmem->rq) 778 return -ENOMEM; 779 780 err = nsim_create_page_pool(&qmem->rq->page_pool, &qmem->rq->napi); 781 if (err) 782 goto err_free; 783 784 if (!ns->rq_reset_mode) 785 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 786 idx); 787 788 return 0; 789 790 err_free: 791 nsim_queue_free(dev, qmem->rq); 792 return err; 793 } 794 795 static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) 796 { 797 struct nsim_queue_mem *qmem = per_queue_mem; 798 struct netdevsim *ns = netdev_priv(dev); 799 800 page_pool_destroy(qmem->pp); 801 if (qmem->rq) { 802 if (!ns->rq_reset_mode) 803 netif_napi_del_locked(&qmem->rq->napi); 804 page_pool_destroy(qmem->rq->page_pool); 805 nsim_queue_free(dev, qmem->rq); 806 } 807 } 808 809 static int 810 nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, 811 void *per_queue_mem, int idx) 812 { 813 struct nsim_queue_mem *qmem = per_queue_mem; 814 struct netdevsim *ns = netdev_priv(dev); 815 816 netdev_assert_locked(dev); 817 818 if (ns->rq_reset_mode == 1) { 819 ns->rq[idx]->page_pool = qmem->pp; 820 napi_enable_locked(&ns->rq[idx]->napi); 821 return 0; 822 } 823 824 /* netif_napi_add()/_del() should normally be called from alloc/free, 825 * here we want to test various call orders. 826 */ 827 if (ns->rq_reset_mode == 2) { 828 netif_napi_del_locked(&ns->rq[idx]->napi); 829 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 830 idx); 831 } else if (ns->rq_reset_mode == 3) { 832 netif_napi_add_config_locked(dev, &qmem->rq->napi, nsim_poll, 833 idx); 834 netif_napi_del_locked(&ns->rq[idx]->napi); 835 } 836 837 ns->rq[idx] = qmem->rq; 838 netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi); 839 napi_enable_locked(&ns->rq[idx]->napi); 840 841 return 0; 842 } 843 844 static int nsim_queue_stop(struct net_device *dev, void *per_queue_mem, int idx) 845 { 846 struct nsim_queue_mem *qmem = per_queue_mem; 847 struct netdevsim *ns = netdev_priv(dev); 848 849 netdev_assert_locked(dev); 850 851 napi_disable_locked(&ns->rq[idx]->napi); 852 853 if (ns->rq_reset_mode == 1) { 854 qmem->pp = ns->rq[idx]->page_pool; 855 page_pool_disable_direct_recycling(qmem->pp); 856 } else { 857 qmem->rq = ns->rq[idx]; 858 } 859 860 return 0; 861 } 862 863 static const struct netdev_queue_mgmt_ops nsim_queue_mgmt_ops = { 864 .ndo_queue_mem_size = sizeof(struct nsim_queue_mem), 865 .ndo_queue_mem_alloc = nsim_queue_mem_alloc, 866 .ndo_queue_mem_free = nsim_queue_mem_free, 867 .ndo_queue_start = nsim_queue_start, 868 .ndo_queue_stop = nsim_queue_stop, 869 }; 870 871 static ssize_t 872 nsim_qreset_write(struct file *file, const char __user *data, 873 size_t count, loff_t *ppos) 874 { 875 struct netdevsim *ns = file->private_data; 876 unsigned int queue, mode; 877 char buf[32]; 878 ssize_t ret; 879 880 if (count >= sizeof(buf)) 881 return -EINVAL; 882 if (copy_from_user(buf, data, count)) 883 return -EFAULT; 884 buf[count] = '\0'; 885 886 ret = sscanf(buf, "%u %u", &queue, &mode); 887 if (ret != 2) 888 return -EINVAL; 889 890 netdev_lock(ns->netdev); 891 if (queue >= ns->netdev->real_num_rx_queues) { 892 ret = -EINVAL; 893 goto exit_unlock; 894 } 895 896 ns->rq_reset_mode = mode; 897 ret = netdev_rx_queue_restart(ns->netdev, queue); 898 ns->rq_reset_mode = 0; 899 if (ret) 900 goto exit_unlock; 901 902 ret = count; 903 exit_unlock: 904 netdev_unlock(ns->netdev); 905 return ret; 906 } 907 908 static const struct file_operations nsim_qreset_fops = { 909 .open = simple_open, 910 .write = nsim_qreset_write, 911 .owner = THIS_MODULE, 912 }; 913 914 static ssize_t 915 nsim_pp_hold_read(struct file *file, char __user *data, 916 size_t count, loff_t *ppos) 917 { 918 struct netdevsim *ns = file->private_data; 919 char buf[3] = "n\n"; 920 921 if (ns->page) 922 buf[0] = 'y'; 923 924 return simple_read_from_buffer(data, count, ppos, buf, 2); 925 } 926 927 static ssize_t 928 nsim_pp_hold_write(struct file *file, const char __user *data, 929 size_t count, loff_t *ppos) 930 { 931 struct netdevsim *ns = file->private_data; 932 ssize_t ret; 933 bool val; 934 935 ret = kstrtobool_from_user(data, count, &val); 936 if (ret) 937 return ret; 938 939 rtnl_lock(); 940 ret = count; 941 if (val == !!ns->page) 942 goto exit; 943 944 if (!netif_running(ns->netdev) && val) { 945 ret = -ENETDOWN; 946 } else if (val) { 947 ns->page = page_pool_dev_alloc_pages(ns->rq[0]->page_pool); 948 if (!ns->page) 949 ret = -ENOMEM; 950 } else { 951 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 952 ns->page, false); 953 ns->page = NULL; 954 } 955 956 exit: 957 rtnl_unlock(); 958 return ret; 959 } 960 961 static const struct file_operations nsim_pp_hold_fops = { 962 .open = simple_open, 963 .read = nsim_pp_hold_read, 964 .write = nsim_pp_hold_write, 965 .llseek = generic_file_llseek, 966 .owner = THIS_MODULE, 967 }; 968 969 static int nsim_vlan_show(struct seq_file *s, void *data) 970 { 971 struct netdevsim *ns = s->private; 972 int vid; 973 974 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 975 seq_printf(s, "ctag %d\n", vid); 976 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 977 seq_printf(s, "stag %d\n", vid); 978 979 return 0; 980 } 981 DEFINE_SHOW_ATTRIBUTE(nsim_vlan); 982 983 static void nsim_setup(struct net_device *dev) 984 { 985 ether_setup(dev); 986 eth_hw_addr_random(dev); 987 988 dev->flags &= ~IFF_MULTICAST; 989 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 990 dev->features |= NETIF_F_HIGHDMA | 991 NETIF_F_SG | 992 NETIF_F_FRAGLIST | 993 NETIF_F_HW_CSUM | 994 NETIF_F_LRO | 995 NETIF_F_TSO | 996 NETIF_F_HW_VLAN_CTAG_FILTER | 997 NETIF_F_HW_VLAN_STAG_FILTER; 998 dev->hw_features |= NETIF_F_HW_TC | 999 NETIF_F_SG | 1000 NETIF_F_FRAGLIST | 1001 NETIF_F_HW_CSUM | 1002 NETIF_F_LRO | 1003 NETIF_F_TSO | 1004 NETIF_F_LOOPBACK | 1005 NETIF_F_HW_VLAN_CTAG_FILTER | 1006 NETIF_F_HW_VLAN_STAG_FILTER; 1007 dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS; 1008 dev->max_mtu = ETH_MAX_MTU; 1009 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_HW_OFFLOAD; 1010 } 1011 1012 static int nsim_queue_init(struct netdevsim *ns) 1013 { 1014 struct net_device *dev = ns->netdev; 1015 int i; 1016 1017 ns->rq = kzalloc_objs(*ns->rq, dev->num_rx_queues, GFP_KERNEL_ACCOUNT); 1018 if (!ns->rq) 1019 return -ENOMEM; 1020 1021 for (i = 0; i < dev->num_rx_queues; i++) { 1022 ns->rq[i] = nsim_queue_alloc(); 1023 if (!ns->rq[i]) 1024 goto err_free_prev; 1025 } 1026 1027 return 0; 1028 1029 err_free_prev: 1030 while (i--) 1031 kfree(ns->rq[i]); 1032 kfree(ns->rq); 1033 return -ENOMEM; 1034 } 1035 1036 static void nsim_queue_uninit(struct netdevsim *ns) 1037 { 1038 struct net_device *dev = ns->netdev; 1039 int i; 1040 1041 for (i = 0; i < dev->num_rx_queues; i++) 1042 nsim_queue_free(dev, ns->rq[i]); 1043 1044 kfree(ns->rq); 1045 ns->rq = NULL; 1046 } 1047 1048 static int nsim_init_netdevsim(struct netdevsim *ns) 1049 { 1050 struct netdevsim *peer; 1051 struct mock_phc *phc; 1052 int err; 1053 1054 phc = mock_phc_create(&ns->nsim_bus_dev->dev); 1055 if (IS_ERR(phc)) 1056 return PTR_ERR(phc); 1057 1058 ns->phc = phc; 1059 ns->netdev->netdev_ops = &nsim_netdev_ops; 1060 ns->netdev->stat_ops = &nsim_stat_ops; 1061 ns->netdev->queue_mgmt_ops = &nsim_queue_mgmt_ops; 1062 netdev_lockdep_set_classes(ns->netdev); 1063 1064 err = nsim_udp_tunnels_info_create(ns->nsim_dev, ns->netdev); 1065 if (err) 1066 goto err_phc_destroy; 1067 1068 rtnl_lock(); 1069 err = nsim_queue_init(ns); 1070 if (err) 1071 goto err_utn_destroy; 1072 1073 err = nsim_bpf_init(ns); 1074 if (err) 1075 goto err_rq_destroy; 1076 1077 nsim_macsec_init(ns); 1078 nsim_ipsec_init(ns); 1079 1080 err = register_netdevice(ns->netdev); 1081 if (err) 1082 goto err_ipsec_teardown; 1083 rtnl_unlock(); 1084 1085 err = nsim_psp_init(ns); 1086 if (err) 1087 goto err_unregister_netdev; 1088 1089 if (IS_ENABLED(CONFIG_DEBUG_NET)) { 1090 ns->nb.notifier_call = netdev_debug_event; 1091 if (register_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1092 &ns->nn)) 1093 ns->nb.notifier_call = NULL; 1094 } 1095 1096 return 0; 1097 1098 err_unregister_netdev: 1099 rtnl_lock(); 1100 peer = rtnl_dereference(ns->peer); 1101 if (peer) 1102 RCU_INIT_POINTER(peer->peer, NULL); 1103 RCU_INIT_POINTER(ns->peer, NULL); 1104 unregister_netdevice(ns->netdev); 1105 err_ipsec_teardown: 1106 nsim_ipsec_teardown(ns); 1107 nsim_macsec_teardown(ns); 1108 nsim_bpf_uninit(ns); 1109 err_rq_destroy: 1110 nsim_queue_uninit(ns); 1111 err_utn_destroy: 1112 rtnl_unlock(); 1113 nsim_udp_tunnels_info_destroy(ns->netdev); 1114 err_phc_destroy: 1115 mock_phc_destroy(ns->phc); 1116 return err; 1117 } 1118 1119 static int nsim_init_netdevsim_vf(struct netdevsim *ns) 1120 { 1121 int err; 1122 1123 ns->netdev->netdev_ops = &nsim_vf_netdev_ops; 1124 rtnl_lock(); 1125 err = register_netdevice(ns->netdev); 1126 rtnl_unlock(); 1127 return err; 1128 } 1129 1130 static void nsim_exit_netdevsim(struct netdevsim *ns) 1131 { 1132 nsim_udp_tunnels_info_destroy(ns->netdev); 1133 mock_phc_destroy(ns->phc); 1134 } 1135 1136 struct netdevsim *nsim_create(struct nsim_dev *nsim_dev, 1137 struct nsim_dev_port *nsim_dev_port, 1138 u8 perm_addr[ETH_ALEN]) 1139 { 1140 struct net_device *dev; 1141 struct netdevsim *ns; 1142 int err; 1143 1144 dev = alloc_netdev_mq(sizeof(*ns), "eth%d", NET_NAME_UNKNOWN, nsim_setup, 1145 nsim_dev->nsim_bus_dev->num_queues); 1146 if (!dev) 1147 return ERR_PTR(-ENOMEM); 1148 1149 if (perm_addr) 1150 memcpy(dev->perm_addr, perm_addr, ETH_ALEN); 1151 1152 dev_net_set(dev, nsim_dev_net(nsim_dev)); 1153 ns = netdev_priv(dev); 1154 ns->netdev = dev; 1155 ns->nsim_dev = nsim_dev; 1156 ns->nsim_dev_port = nsim_dev_port; 1157 ns->nsim_bus_dev = nsim_dev->nsim_bus_dev; 1158 SET_NETDEV_DEV(dev, &ns->nsim_bus_dev->dev); 1159 SET_NETDEV_DEVLINK_PORT(dev, &nsim_dev_port->devlink_port); 1160 nsim_ethtool_init(ns); 1161 if (nsim_dev_port_is_pf(nsim_dev_port)) 1162 err = nsim_init_netdevsim(ns); 1163 else 1164 err = nsim_init_netdevsim_vf(ns); 1165 if (err) 1166 goto err_free_netdev; 1167 1168 ns->pp_dfs = debugfs_create_file("pp_hold", 0600, nsim_dev_port->ddir, 1169 ns, &nsim_pp_hold_fops); 1170 ns->qr_dfs = debugfs_create_file("queue_reset", 0200, 1171 nsim_dev_port->ddir, ns, 1172 &nsim_qreset_fops); 1173 ns->vlan_dfs = debugfs_create_file("vlan", 0400, nsim_dev_port->ddir, 1174 ns, &nsim_vlan_fops); 1175 return ns; 1176 1177 err_free_netdev: 1178 nsim_ethtool_fini(ns); 1179 free_netdev(dev); 1180 return ERR_PTR(err); 1181 } 1182 1183 void nsim_destroy(struct netdevsim *ns) 1184 { 1185 struct net_device *dev = ns->netdev; 1186 struct netdevsim *peer; 1187 u16 vid; 1188 1189 debugfs_remove(ns->vlan_dfs); 1190 debugfs_remove(ns->qr_dfs); 1191 debugfs_remove(ns->pp_dfs); 1192 nsim_ethtool_fini(ns); 1193 1194 if (ns->nb.notifier_call) 1195 unregister_netdevice_notifier_dev_net(ns->netdev, &ns->nb, 1196 &ns->nn); 1197 1198 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1199 nsim_psp_uninit(ns); 1200 1201 rtnl_lock(); 1202 peer = rtnl_dereference(ns->peer); 1203 if (peer) 1204 RCU_INIT_POINTER(peer->peer, NULL); 1205 RCU_INIT_POINTER(ns->peer, NULL); 1206 unregister_netdevice(dev); 1207 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) { 1208 nsim_macsec_teardown(ns); 1209 nsim_ipsec_teardown(ns); 1210 nsim_bpf_uninit(ns); 1211 nsim_queue_uninit(ns); 1212 } 1213 rtnl_unlock(); 1214 if (nsim_dev_port_is_pf(ns->nsim_dev_port)) 1215 nsim_exit_netdevsim(ns); 1216 1217 for_each_set_bit(vid, ns->vlan.ctag, VLAN_N_VID) 1218 WARN_ON_ONCE(1); 1219 for_each_set_bit(vid, ns->vlan.stag, VLAN_N_VID) 1220 WARN_ON_ONCE(1); 1221 1222 /* Put this intentionally late to exercise the orphaning path */ 1223 if (ns->page) { 1224 page_pool_put_full_page(pp_page_to_nmdesc(ns->page)->pp, 1225 ns->page, false); 1226 ns->page = NULL; 1227 } 1228 1229 free_netdev(dev); 1230 } 1231 1232 bool netdev_is_nsim(struct net_device *dev) 1233 { 1234 return dev->netdev_ops == &nsim_netdev_ops; 1235 } 1236 1237 static int nsim_validate(struct nlattr *tb[], struct nlattr *data[], 1238 struct netlink_ext_ack *extack) 1239 { 1240 NL_SET_ERR_MSG_MOD(extack, 1241 "Please use: echo \"[ID] [PORT_COUNT] [NUM_QUEUES]\" > /sys/bus/netdevsim/new_device"); 1242 return -EOPNOTSUPP; 1243 } 1244 1245 static struct rtnl_link_ops nsim_link_ops __read_mostly = { 1246 .kind = DRV_NAME, 1247 .validate = nsim_validate, 1248 }; 1249 1250 static int __init nsim_module_init(void) 1251 { 1252 int err; 1253 1254 err = nsim_dev_init(); 1255 if (err) 1256 return err; 1257 1258 err = nsim_bus_init(); 1259 if (err) 1260 goto err_dev_exit; 1261 1262 err = rtnl_link_register(&nsim_link_ops); 1263 if (err) 1264 goto err_bus_exit; 1265 1266 return 0; 1267 1268 err_bus_exit: 1269 nsim_bus_exit(); 1270 err_dev_exit: 1271 nsim_dev_exit(); 1272 return err; 1273 } 1274 1275 static void __exit nsim_module_exit(void) 1276 { 1277 rtnl_link_unregister(&nsim_link_ops); 1278 nsim_bus_exit(); 1279 nsim_dev_exit(); 1280 } 1281 1282 module_init(nsim_module_init); 1283 module_exit(nsim_module_exit); 1284 MODULE_LICENSE("GPL"); 1285 MODULE_DESCRIPTION("Simulated networking device for testing"); 1286 MODULE_ALIAS_RTNL_LINK(DRV_NAME); 1287