1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause 2 /* 3 * PCIe NTB Network Linux driver 4 */ 5 #include <linux/etherdevice.h> 6 #include <linux/ethtool.h> 7 #include <linux/module.h> 8 #include <linux/pci.h> 9 #include <linux/ntb.h> 10 #include <linux/ntb_transport.h> 11 #include <linux/slab.h> 12 13 #define NTB_NETDEV_VER "0.7" 14 15 MODULE_DESCRIPTION(KBUILD_MODNAME); 16 MODULE_VERSION(NTB_NETDEV_VER); 17 MODULE_LICENSE("Dual BSD/GPL"); 18 MODULE_AUTHOR("Intel Corporation"); 19 20 /* Time in usecs for tx resource reaper */ 21 static unsigned int tx_time = 1; 22 23 /* Number of descriptors to free before resuming tx */ 24 static unsigned int tx_start = 10; 25 26 /* Number of descriptors still available before stop upper layer tx */ 27 static unsigned int tx_stop = 5; 28 29 #define NTB_NETDEV_MAX_QUEUES 64 30 #define NTB_NETDEV_DEFAULT_QUEUES 1 31 32 struct ntb_netdev; 33 34 struct ntb_netdev_queue { 35 struct ntb_netdev *ntdev; 36 struct ntb_transport_qp *qp; 37 struct timer_list tx_timer; 38 u16 qid; 39 }; 40 41 struct ntb_netdev { 42 struct pci_dev *pdev; 43 struct device *client_dev; 44 struct net_device *ndev; 45 unsigned int num_queues; 46 struct ntb_netdev_queue *queues; 47 }; 48 49 #define NTB_TX_TIMEOUT_MS 1000 50 #define NTB_RXQ_SIZE 100 51 52 static void ntb_netdev_update_carrier(struct ntb_netdev *dev) 53 { 54 struct net_device *ndev; 55 bool any_up = false; 56 unsigned int i; 57 58 ndev = dev->ndev; 59 60 for (i = 0; i < dev->num_queues; i++) { 61 if (ntb_transport_link_query(dev->queues[i].qp)) { 62 any_up = true; 63 break; 64 } 65 } 66 67 if (any_up) 68 netif_carrier_on(ndev); 69 else 70 netif_carrier_off(ndev); 71 } 72 73 static void ntb_netdev_queue_rx_drain(struct ntb_netdev_queue *queue) 74 { 75 struct sk_buff *skb; 76 int len; 77 78 while ((skb = ntb_transport_rx_remove(queue->qp, &len))) 79 dev_kfree_skb(skb); 80 } 81 82 static int ntb_netdev_queue_rx_fill(struct net_device *ndev, 83 struct ntb_netdev_queue *queue) 84 { 85 struct sk_buff *skb; 86 int rc, i; 87 88 for (i = 0; i < NTB_RXQ_SIZE; i++) { 89 skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); 90 if (!skb) 91 return -ENOMEM; 92 93 rc = ntb_transport_rx_enqueue(queue->qp, skb, skb->data, 94 ndev->mtu + ETH_HLEN); 95 if (rc) { 96 dev_kfree_skb(skb); 97 return rc; 98 } 99 } 100 101 return 0; 102 } 103 104 static void ntb_netdev_event_handler(void *data, int link_is_up) 105 { 106 struct ntb_netdev_queue *q = data; 107 struct ntb_netdev *dev = q->ntdev; 108 struct net_device *ndev; 109 110 ndev = dev->ndev; 111 112 netdev_dbg(ndev, "Event %x, Link %x, qp %u\n", link_is_up, 113 ntb_transport_link_query(q->qp), q->qid); 114 115 if (netif_running(ndev)) { 116 if (link_is_up) 117 netif_wake_subqueue(ndev, q->qid); 118 else 119 netif_stop_subqueue(ndev, q->qid); 120 } 121 122 ntb_netdev_update_carrier(dev); 123 } 124 125 static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 126 void *data, int len) 127 { 128 struct ntb_netdev_queue *q = qp_data; 129 struct ntb_netdev *dev = q->ntdev; 130 struct sk_buff *skb, *new_skb; 131 struct net_device *ndev; 132 int rc; 133 134 ndev = dev->ndev; 135 skb = data; 136 if (!skb) 137 return; 138 139 netdev_dbg(ndev, "%s: %d byte payload received\n", __func__, len); 140 141 if (len < 0) { 142 ndev->stats.rx_errors++; 143 ndev->stats.rx_length_errors++; 144 goto enqueue_again; 145 } 146 147 new_skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); 148 if (!new_skb) { 149 ndev->stats.rx_dropped++; 150 goto enqueue_again; 151 } 152 153 skb_put(skb, len); 154 skb->protocol = eth_type_trans(skb, ndev); 155 skb->ip_summed = CHECKSUM_NONE; 156 skb_record_rx_queue(skb, q->qid); 157 158 if (netif_rx(skb) == NET_RX_DROP) { 159 ndev->stats.rx_errors++; 160 ndev->stats.rx_dropped++; 161 } else { 162 ndev->stats.rx_packets++; 163 ndev->stats.rx_bytes += len; 164 } 165 166 skb = new_skb; 167 168 enqueue_again: 169 rc = ntb_transport_rx_enqueue(qp, skb, skb->data, ndev->mtu + ETH_HLEN); 170 if (rc) { 171 dev_kfree_skb_any(skb); 172 ndev->stats.rx_errors++; 173 ndev->stats.rx_fifo_errors++; 174 } 175 } 176 177 static int __ntb_netdev_maybe_stop_tx(struct net_device *netdev, 178 struct ntb_netdev_queue *q, int size) 179 { 180 netif_stop_subqueue(netdev, q->qid); 181 182 /* Make sure to see the latest value of ntb_transport_tx_free_entry() 183 * since the queue was last started. 184 */ 185 smp_mb(); 186 187 if (likely(ntb_transport_tx_free_entry(q->qp) < size)) { 188 mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time)); 189 return -EBUSY; 190 } 191 192 /* The subqueue must be kept stopped if the link is down */ 193 if (ntb_transport_link_query(q->qp)) 194 netif_start_subqueue(netdev, q->qid); 195 196 return 0; 197 } 198 199 static int ntb_netdev_maybe_stop_tx(struct net_device *ndev, 200 struct ntb_netdev_queue *q, int size) 201 { 202 if (__netif_subqueue_stopped(ndev, q->qid) || 203 (ntb_transport_tx_free_entry(q->qp) >= size)) 204 return 0; 205 206 return __ntb_netdev_maybe_stop_tx(ndev, q, size); 207 } 208 209 static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 210 void *data, int len) 211 { 212 struct ntb_netdev_queue *q = qp_data; 213 struct ntb_netdev *dev = q->ntdev; 214 struct net_device *ndev; 215 struct sk_buff *skb; 216 217 ndev = dev->ndev; 218 skb = data; 219 if (!skb || !ndev) 220 return; 221 222 if (len > 0) { 223 ndev->stats.tx_packets++; 224 ndev->stats.tx_bytes += skb->len; 225 } else { 226 ndev->stats.tx_errors++; 227 ndev->stats.tx_aborted_errors++; 228 } 229 230 dev_kfree_skb_any(skb); 231 232 if (ntb_transport_tx_free_entry(qp) >= tx_start) { 233 /* Make sure anybody stopping the queue after this sees the new 234 * value of ntb_transport_tx_free_entry() 235 */ 236 smp_mb(); 237 if (__netif_subqueue_stopped(ndev, q->qid) && 238 ntb_transport_link_query(q->qp)) 239 netif_wake_subqueue(ndev, q->qid); 240 } 241 } 242 243 static const struct ntb_queue_handlers ntb_netdev_handlers = { 244 .tx_handler = ntb_netdev_tx_handler, 245 .rx_handler = ntb_netdev_rx_handler, 246 .event_handler = ntb_netdev_event_handler, 247 }; 248 249 static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb, 250 struct net_device *ndev) 251 { 252 struct ntb_netdev *dev = netdev_priv(ndev); 253 u16 qid = skb_get_queue_mapping(skb); 254 struct ntb_netdev_queue *q; 255 int rc; 256 257 q = &dev->queues[qid]; 258 259 ntb_netdev_maybe_stop_tx(ndev, q, tx_stop); 260 261 rc = ntb_transport_tx_enqueue(q->qp, skb, skb->data, skb->len); 262 if (rc) 263 goto err; 264 265 /* check for next submit */ 266 ntb_netdev_maybe_stop_tx(ndev, q, tx_stop); 267 268 return NETDEV_TX_OK; 269 270 err: 271 ndev->stats.tx_dropped++; 272 ndev->stats.tx_errors++; 273 return NETDEV_TX_BUSY; 274 } 275 276 static void ntb_netdev_tx_timer(struct timer_list *t) 277 { 278 struct ntb_netdev_queue *q = timer_container_of(q, t, tx_timer); 279 struct ntb_netdev *dev = q->ntdev; 280 struct net_device *ndev; 281 282 ndev = dev->ndev; 283 284 if (ntb_transport_tx_free_entry(q->qp) < tx_stop) { 285 mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time)); 286 } else { 287 /* Make sure anybody stopping the queue after this sees the new 288 * value of ntb_transport_tx_free_entry() 289 */ 290 smp_mb(); 291 292 /* The subqueue must be kept stopped if the link is down */ 293 if (__netif_subqueue_stopped(ndev, q->qid) && 294 ntb_transport_link_query(q->qp)) 295 netif_wake_subqueue(ndev, q->qid); 296 } 297 } 298 299 static int ntb_netdev_open(struct net_device *ndev) 300 { 301 struct ntb_netdev *dev = netdev_priv(ndev); 302 struct ntb_netdev_queue *queue; 303 unsigned int q; 304 int rc = 0; 305 306 /* Add some empty rx bufs for each queue */ 307 for (q = 0; q < dev->num_queues; q++) { 308 queue = &dev->queues[q]; 309 310 rc = ntb_netdev_queue_rx_fill(ndev, queue); 311 if (rc) 312 goto err; 313 314 timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0); 315 } 316 317 netif_carrier_off(ndev); 318 netif_tx_stop_all_queues(ndev); 319 320 for (q = 0; q < dev->num_queues; q++) 321 ntb_transport_link_up(dev->queues[q].qp); 322 323 return 0; 324 325 err: 326 for (q = 0; q < dev->num_queues; q++) { 327 queue = &dev->queues[q]; 328 ntb_netdev_queue_rx_drain(queue); 329 } 330 return rc; 331 } 332 333 static int ntb_netdev_close(struct net_device *ndev) 334 { 335 struct ntb_netdev *dev = netdev_priv(ndev); 336 struct ntb_netdev_queue *queue; 337 unsigned int q; 338 339 netif_tx_stop_all_queues(ndev); 340 netif_carrier_off(ndev); 341 342 for (q = 0; q < dev->num_queues; q++) { 343 queue = &dev->queues[q]; 344 345 ntb_transport_link_down(queue->qp); 346 ntb_netdev_queue_rx_drain(queue); 347 timer_delete_sync(&queue->tx_timer); 348 } 349 350 return 0; 351 } 352 353 static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu) 354 { 355 struct ntb_netdev *dev = netdev_priv(ndev); 356 struct ntb_netdev_queue *queue; 357 struct sk_buff *skb; 358 unsigned int q, i; 359 int len, rc = 0; 360 361 if (new_mtu > ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN) 362 return -EINVAL; 363 364 if (!netif_running(ndev)) { 365 WRITE_ONCE(ndev->mtu, new_mtu); 366 return 0; 367 } 368 369 /* Bring down the link and dispose of posted rx entries */ 370 for (q = 0; q < dev->num_queues; q++) 371 ntb_transport_link_down(dev->queues[q].qp); 372 373 if (ndev->mtu < new_mtu) { 374 for (q = 0; q < dev->num_queues; q++) { 375 queue = &dev->queues[q]; 376 377 for (i = 0; 378 (skb = ntb_transport_rx_remove(queue->qp, &len)); 379 i++) 380 dev_kfree_skb(skb); 381 382 for (; i; i--) { 383 skb = netdev_alloc_skb(ndev, 384 new_mtu + ETH_HLEN); 385 if (!skb) { 386 rc = -ENOMEM; 387 goto err; 388 } 389 390 rc = ntb_transport_rx_enqueue(queue->qp, skb, 391 skb->data, 392 new_mtu + 393 ETH_HLEN); 394 if (rc) { 395 dev_kfree_skb(skb); 396 goto err; 397 } 398 } 399 } 400 } 401 402 WRITE_ONCE(ndev->mtu, new_mtu); 403 404 for (q = 0; q < dev->num_queues; q++) 405 ntb_transport_link_up(dev->queues[q].qp); 406 407 return 0; 408 409 err: 410 for (q = 0; q < dev->num_queues; q++) { 411 struct ntb_netdev_queue *queue = &dev->queues[q]; 412 413 ntb_transport_link_down(queue->qp); 414 415 ntb_netdev_queue_rx_drain(queue); 416 } 417 418 netdev_err(ndev, "Error changing MTU, device inoperable\n"); 419 return rc; 420 } 421 422 static const struct net_device_ops ntb_netdev_ops = { 423 .ndo_open = ntb_netdev_open, 424 .ndo_stop = ntb_netdev_close, 425 .ndo_start_xmit = ntb_netdev_start_xmit, 426 .ndo_change_mtu = ntb_netdev_change_mtu, 427 .ndo_set_mac_address = eth_mac_addr, 428 }; 429 430 static void ntb_get_drvinfo(struct net_device *ndev, 431 struct ethtool_drvinfo *info) 432 { 433 struct ntb_netdev *dev = netdev_priv(ndev); 434 435 strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 436 strscpy(info->version, NTB_NETDEV_VER, sizeof(info->version)); 437 strscpy(info->bus_info, pci_name(dev->pdev), sizeof(info->bus_info)); 438 } 439 440 static int ntb_get_link_ksettings(struct net_device *dev, 441 struct ethtool_link_ksettings *cmd) 442 { 443 ethtool_link_ksettings_zero_link_mode(cmd, supported); 444 ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane); 445 ethtool_link_ksettings_zero_link_mode(cmd, advertising); 446 ethtool_link_ksettings_add_link_mode(cmd, advertising, Backplane); 447 448 cmd->base.speed = SPEED_UNKNOWN; 449 cmd->base.duplex = DUPLEX_FULL; 450 cmd->base.port = PORT_OTHER; 451 cmd->base.phy_address = 0; 452 cmd->base.autoneg = AUTONEG_ENABLE; 453 454 return 0; 455 } 456 457 static void ntb_get_channels(struct net_device *ndev, 458 struct ethtool_channels *channels) 459 { 460 struct ntb_netdev *dev = netdev_priv(ndev); 461 462 channels->combined_count = dev->num_queues; 463 channels->max_combined = ndev->num_tx_queues; 464 } 465 466 static int ntb_inc_channels(struct net_device *ndev, 467 unsigned int old, unsigned int new) 468 { 469 struct ntb_netdev *dev = netdev_priv(ndev); 470 bool running = netif_running(ndev); 471 struct ntb_netdev_queue *queue; 472 unsigned int q, created; 473 int rc; 474 475 created = old; 476 for (q = old; q < new; q++) { 477 queue = &dev->queues[q]; 478 479 queue->ntdev = dev; 480 queue->qid = q; 481 queue->qp = ntb_transport_create_queue(queue, dev->client_dev, 482 &ntb_netdev_handlers); 483 if (!queue->qp) { 484 rc = -ENOSPC; 485 goto err_new; 486 } 487 created++; 488 489 if (!running) 490 continue; 491 492 timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0); 493 494 rc = ntb_netdev_queue_rx_fill(ndev, queue); 495 if (rc) 496 goto err_new; 497 498 /* 499 * Carrier may already be on due to other QPs. Keep the new 500 * subqueue stopped until we get a Link Up event for this QP. 501 */ 502 netif_stop_subqueue(ndev, q); 503 } 504 505 rc = netif_set_real_num_queues(ndev, new, new); 506 if (rc) 507 goto err_new; 508 509 dev->num_queues = new; 510 511 if (running) 512 for (q = old; q < new; q++) 513 ntb_transport_link_up(dev->queues[q].qp); 514 515 return 0; 516 517 err_new: 518 if (running) { 519 unsigned int rollback = created; 520 521 while (rollback-- > old) { 522 queue = &dev->queues[rollback]; 523 ntb_transport_link_down(queue->qp); 524 ntb_netdev_queue_rx_drain(queue); 525 timer_delete_sync(&queue->tx_timer); 526 } 527 } 528 while (created-- > old) { 529 queue = &dev->queues[created]; 530 ntb_transport_free_queue(queue->qp); 531 queue->qp = NULL; 532 } 533 return rc; 534 } 535 536 static int ntb_dec_channels(struct net_device *ndev, 537 unsigned int old, unsigned int new) 538 { 539 struct ntb_netdev *dev = netdev_priv(ndev); 540 bool running = netif_running(ndev); 541 struct ntb_netdev_queue *queue; 542 unsigned int q; 543 int rc; 544 545 if (running) 546 for (q = new; q < old; q++) 547 netif_stop_subqueue(ndev, q); 548 549 rc = netif_set_real_num_queues(ndev, new, new); 550 if (rc) 551 goto err; 552 553 /* Publish new queue count before invalidating QP pointers */ 554 dev->num_queues = new; 555 556 for (q = new; q < old; q++) { 557 queue = &dev->queues[q]; 558 559 if (running) { 560 ntb_transport_link_down(queue->qp); 561 ntb_netdev_queue_rx_drain(queue); 562 timer_delete_sync(&queue->tx_timer); 563 } 564 565 ntb_transport_free_queue(queue->qp); 566 queue->qp = NULL; 567 } 568 569 /* 570 * It might be the case that the removed queues are the only queues that 571 * were up, so see if the global carrier needs to change. 572 */ 573 ntb_netdev_update_carrier(dev); 574 return 0; 575 576 err: 577 if (running) { 578 for (q = new; q < old; q++) 579 netif_wake_subqueue(ndev, q); 580 } 581 return rc; 582 } 583 584 static int ntb_set_channels(struct net_device *ndev, 585 struct ethtool_channels *channels) 586 { 587 struct ntb_netdev *dev = netdev_priv(ndev); 588 unsigned int new = channels->combined_count; 589 unsigned int old = dev->num_queues; 590 591 if (new == old) 592 return 0; 593 594 if (new < old) 595 return ntb_dec_channels(ndev, old, new); 596 else 597 return ntb_inc_channels(ndev, old, new); 598 } 599 600 static const struct ethtool_ops ntb_ethtool_ops = { 601 .get_drvinfo = ntb_get_drvinfo, 602 .get_link = ethtool_op_get_link, 603 .get_link_ksettings = ntb_get_link_ksettings, 604 .get_channels = ntb_get_channels, 605 .set_channels = ntb_set_channels, 606 }; 607 608 static int ntb_netdev_probe(struct device *client_dev) 609 { 610 struct ntb_dev *ntb; 611 struct net_device *ndev; 612 struct pci_dev *pdev; 613 struct ntb_netdev *dev; 614 unsigned int q; 615 int rc; 616 617 ntb = dev_ntb(client_dev->parent); 618 pdev = ntb->pdev; 619 if (!pdev) 620 return -ENODEV; 621 622 ndev = alloc_etherdev_mq(sizeof(*dev), NTB_NETDEV_MAX_QUEUES); 623 if (!ndev) 624 return -ENOMEM; 625 626 SET_NETDEV_DEV(ndev, client_dev); 627 628 dev = netdev_priv(ndev); 629 dev->ndev = ndev; 630 dev->pdev = pdev; 631 dev->client_dev = client_dev; 632 dev->num_queues = 0; 633 634 dev->queues = kzalloc_objs(*dev->queues, NTB_NETDEV_MAX_QUEUES, 635 GFP_KERNEL); 636 if (!dev->queues) { 637 rc = -ENOMEM; 638 goto err_free_netdev; 639 } 640 641 ndev->features = NETIF_F_HIGHDMA; 642 643 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 644 645 ndev->hw_features = ndev->features; 646 ndev->watchdog_timeo = msecs_to_jiffies(NTB_TX_TIMEOUT_MS); 647 648 eth_random_addr(ndev->perm_addr); 649 dev_addr_set(ndev, ndev->perm_addr); 650 651 ndev->netdev_ops = &ntb_netdev_ops; 652 ndev->ethtool_ops = &ntb_ethtool_ops; 653 654 ndev->min_mtu = 0; 655 ndev->max_mtu = ETH_MAX_MTU; 656 657 for (q = 0; q < NTB_NETDEV_DEFAULT_QUEUES; q++) { 658 struct ntb_netdev_queue *queue = &dev->queues[q]; 659 660 queue->ntdev = dev; 661 queue->qid = q; 662 queue->qp = ntb_transport_create_queue(queue, client_dev, 663 &ntb_netdev_handlers); 664 if (!queue->qp) 665 break; 666 667 dev->num_queues++; 668 } 669 670 if (!dev->num_queues) { 671 rc = -EIO; 672 goto err_free_queues; 673 } 674 675 rc = netif_set_real_num_queues(ndev, dev->num_queues, dev->num_queues); 676 if (rc) 677 goto err_free_qps; 678 679 ndev->mtu = ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN; 680 681 rc = register_netdev(ndev); 682 if (rc) 683 goto err_free_qps; 684 685 dev_set_drvdata(client_dev, ndev); 686 dev_info(&pdev->dev, "%s created with %u queue pairs\n", 687 ndev->name, dev->num_queues); 688 return 0; 689 690 err_free_qps: 691 for (q = 0; q < dev->num_queues; q++) 692 ntb_transport_free_queue(dev->queues[q].qp); 693 694 err_free_queues: 695 kfree(dev->queues); 696 697 err_free_netdev: 698 free_netdev(ndev); 699 return rc; 700 } 701 702 static void ntb_netdev_remove(struct device *client_dev) 703 { 704 struct net_device *ndev = dev_get_drvdata(client_dev); 705 struct ntb_netdev *dev = netdev_priv(ndev); 706 unsigned int q; 707 708 unregister_netdev(ndev); 709 for (q = 0; q < dev->num_queues; q++) 710 ntb_transport_free_queue(dev->queues[q].qp); 711 712 kfree(dev->queues); 713 free_netdev(ndev); 714 } 715 716 static struct ntb_transport_client ntb_netdev_client = { 717 .driver.name = KBUILD_MODNAME, 718 .driver.owner = THIS_MODULE, 719 .probe = ntb_netdev_probe, 720 .remove = ntb_netdev_remove, 721 }; 722 723 static int __init ntb_netdev_init_module(void) 724 { 725 int rc; 726 727 rc = ntb_transport_register_client_dev(KBUILD_MODNAME); 728 if (rc) 729 return rc; 730 731 rc = ntb_transport_register_client(&ntb_netdev_client); 732 if (rc) { 733 ntb_transport_unregister_client_dev(KBUILD_MODNAME); 734 return rc; 735 } 736 737 return 0; 738 } 739 late_initcall(ntb_netdev_init_module); 740 741 static void __exit ntb_netdev_exit_module(void) 742 { 743 ntb_transport_unregister_client(&ntb_netdev_client); 744 ntb_transport_unregister_client_dev(KBUILD_MODNAME); 745 } 746 module_exit(ntb_netdev_exit_module); 747