1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause 2 /* 3 * PCIe NTB Network Linux driver 4 */ 5 #include <linux/etherdevice.h> 6 #include <linux/ethtool.h> 7 #include <linux/module.h> 8 #include <linux/pci.h> 9 #include <linux/ntb.h> 10 #include <linux/ntb_transport.h> 11 #include <linux/slab.h> 12 13 #define NTB_NETDEV_VER "0.7" 14 15 MODULE_DESCRIPTION(KBUILD_MODNAME); 16 MODULE_VERSION(NTB_NETDEV_VER); 17 MODULE_LICENSE("Dual BSD/GPL"); 18 MODULE_AUTHOR("Intel Corporation"); 19 20 /* Time in usecs for tx resource reaper */ 21 static unsigned int tx_time = 1; 22 23 /* Number of descriptors to free before resuming tx */ 24 static unsigned int tx_start = 10; 25 26 /* Number of descriptors still available before stop upper layer tx */ 27 static unsigned int tx_stop = 5; 28 29 #define NTB_NETDEV_MAX_QUEUES 64 30 #define NTB_NETDEV_DEFAULT_QUEUES 1 31 32 struct ntb_netdev; 33 34 struct ntb_netdev_queue { 35 struct ntb_netdev *ntdev; 36 struct ntb_transport_qp *qp; 37 struct timer_list tx_timer; 38 u16 qid; 39 }; 40 41 struct ntb_netdev { 42 struct pci_dev *pdev; 43 struct device *client_dev; 44 struct net_device *ndev; 45 unsigned int num_queues; 46 struct ntb_netdev_queue *queues; 47 }; 48 49 #define NTB_TX_TIMEOUT_MS 1000 50 #define NTB_RXQ_SIZE 100 51 52 static void ntb_netdev_update_carrier(struct ntb_netdev *dev) 53 { 54 struct net_device *ndev; 55 bool any_up = false; 56 unsigned int i; 57 58 ndev = dev->ndev; 59 60 for (i = 0; i < dev->num_queues; i++) { 61 if (ntb_transport_link_query(dev->queues[i].qp)) { 62 any_up = true; 63 break; 64 } 65 } 66 67 if (any_up) 68 netif_carrier_on(ndev); 69 else 70 netif_carrier_off(ndev); 71 } 72 73 static void ntb_netdev_queue_rx_drain(struct ntb_netdev_queue *queue) 74 { 75 struct sk_buff *skb; 76 int len; 77 78 while ((skb = ntb_transport_rx_remove(queue->qp, &len))) 79 dev_kfree_skb(skb); 80 } 81 82 static int ntb_netdev_queue_rx_fill(struct net_device *ndev, 83 struct ntb_netdev_queue *queue) 84 { 85 struct sk_buff *skb; 86 int rc, i; 87 88 for (i = 0; i < NTB_RXQ_SIZE; i++) { 89 skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); 90 if (!skb) 91 return -ENOMEM; 92 93 rc = ntb_transport_rx_enqueue(queue->qp, skb, skb->data, 94 ndev->mtu + ETH_HLEN); 95 if (rc) { 96 dev_kfree_skb(skb); 97 return rc; 98 } 99 } 100 101 return 0; 102 } 103 104 static void ntb_netdev_event_handler(void *data, int link_is_up) 105 { 106 struct ntb_netdev_queue *q = data; 107 struct ntb_netdev *dev = q->ntdev; 108 struct net_device *ndev; 109 110 ndev = dev->ndev; 111 112 netdev_dbg(ndev, "Event %x, Link %x, qp %u\n", link_is_up, 113 ntb_transport_link_query(q->qp), q->qid); 114 115 if (netif_running(ndev)) { 116 if (link_is_up) 117 netif_wake_subqueue(ndev, q->qid); 118 else 119 netif_stop_subqueue(ndev, q->qid); 120 } 121 122 ntb_netdev_update_carrier(dev); 123 } 124 125 static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 126 void *data, int len) 127 { 128 struct ntb_netdev_queue *q = qp_data; 129 struct ntb_netdev *dev = q->ntdev; 130 struct sk_buff *skb, *new_skb; 131 struct net_device *ndev; 132 int rc; 133 134 ndev = dev->ndev; 135 skb = data; 136 if (!skb) 137 return; 138 139 netdev_dbg(ndev, "%s: %d byte payload received\n", __func__, len); 140 141 if (len < 0) { 142 ndev->stats.rx_errors++; 143 ndev->stats.rx_length_errors++; 144 goto enqueue_again; 145 } 146 147 ndev->stats.rx_packets++; 148 ndev->stats.rx_bytes += len; 149 150 new_skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); 151 if (!new_skb) { 152 ndev->stats.rx_dropped++; 153 goto enqueue_again; 154 } 155 156 skb_put(skb, len); 157 skb->protocol = eth_type_trans(skb, ndev); 158 skb->ip_summed = CHECKSUM_NONE; 159 skb_record_rx_queue(skb, q->qid); 160 161 netif_rx(skb); 162 163 skb = new_skb; 164 165 enqueue_again: 166 rc = ntb_transport_rx_enqueue(qp, skb, skb->data, ndev->mtu + ETH_HLEN); 167 if (rc) { 168 dev_kfree_skb_any(skb); 169 ndev->stats.rx_errors++; 170 ndev->stats.rx_fifo_errors++; 171 } 172 } 173 174 static int __ntb_netdev_maybe_stop_tx(struct net_device *netdev, 175 struct ntb_netdev_queue *q, int size) 176 { 177 netif_stop_subqueue(netdev, q->qid); 178 179 /* Make sure to see the latest value of ntb_transport_tx_free_entry() 180 * since the queue was last started. 181 */ 182 smp_mb(); 183 184 if (likely(ntb_transport_tx_free_entry(q->qp) < size)) { 185 mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time)); 186 return -EBUSY; 187 } 188 189 /* The subqueue must be kept stopped if the link is down */ 190 if (ntb_transport_link_query(q->qp)) 191 netif_start_subqueue(netdev, q->qid); 192 193 return 0; 194 } 195 196 static int ntb_netdev_maybe_stop_tx(struct net_device *ndev, 197 struct ntb_netdev_queue *q, int size) 198 { 199 if (__netif_subqueue_stopped(ndev, q->qid)) 200 return -EBUSY; 201 202 if (ntb_transport_tx_free_entry(q->qp) >= size) 203 return 0; 204 205 return __ntb_netdev_maybe_stop_tx(ndev, q, size); 206 } 207 208 static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 209 void *data, int len) 210 { 211 struct ntb_netdev_queue *q = qp_data; 212 struct ntb_netdev *dev = q->ntdev; 213 struct net_device *ndev; 214 struct sk_buff *skb; 215 216 ndev = dev->ndev; 217 skb = data; 218 if (!skb || !ndev) 219 return; 220 221 if (len > 0) { 222 ndev->stats.tx_packets++; 223 ndev->stats.tx_bytes += skb->len; 224 } else { 225 ndev->stats.tx_errors++; 226 ndev->stats.tx_aborted_errors++; 227 } 228 229 dev_kfree_skb_any(skb); 230 231 if (ntb_transport_tx_free_entry(qp) >= tx_start) { 232 /* Make sure anybody stopping the queue after this sees the new 233 * value of ntb_transport_tx_free_entry() 234 */ 235 smp_mb(); 236 if (__netif_subqueue_stopped(ndev, q->qid) && 237 ntb_transport_link_query(q->qp)) 238 netif_wake_subqueue(ndev, q->qid); 239 } 240 } 241 242 static const struct ntb_queue_handlers ntb_netdev_handlers = { 243 .tx_handler = ntb_netdev_tx_handler, 244 .rx_handler = ntb_netdev_rx_handler, 245 .event_handler = ntb_netdev_event_handler, 246 }; 247 248 static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb, 249 struct net_device *ndev) 250 { 251 struct ntb_netdev *dev = netdev_priv(ndev); 252 u16 qid = skb_get_queue_mapping(skb); 253 struct ntb_netdev_queue *q; 254 int rc; 255 256 q = &dev->queues[qid]; 257 258 if (unlikely(ntb_netdev_maybe_stop_tx(ndev, q, tx_stop))) 259 return NETDEV_TX_BUSY; 260 261 rc = ntb_transport_tx_enqueue(q->qp, skb, skb->data, skb->len); 262 if (rc) { 263 if (rc == -EAGAIN || rc == -EBUSY) { 264 netif_stop_subqueue(ndev, q->qid); 265 mod_timer(&q->tx_timer, 266 jiffies + usecs_to_jiffies(tx_time)); 267 return NETDEV_TX_BUSY; 268 } 269 270 goto drop; 271 } 272 273 /* check for next submit */ 274 ntb_netdev_maybe_stop_tx(ndev, q, tx_stop); 275 276 return NETDEV_TX_OK; 277 278 drop: 279 dev_kfree_skb_any(skb); 280 ndev->stats.tx_dropped++; 281 return NETDEV_TX_OK; 282 } 283 284 static void ntb_netdev_tx_timer(struct timer_list *t) 285 { 286 struct ntb_netdev_queue *q = timer_container_of(q, t, tx_timer); 287 struct ntb_netdev *dev = q->ntdev; 288 struct net_device *ndev; 289 290 ndev = dev->ndev; 291 292 if (ntb_transport_tx_free_entry(q->qp) < tx_stop) { 293 mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time)); 294 } else { 295 /* Make sure anybody stopping the queue after this sees the new 296 * value of ntb_transport_tx_free_entry() 297 */ 298 smp_mb(); 299 300 /* The subqueue must be kept stopped if the link is down */ 301 if (__netif_subqueue_stopped(ndev, q->qid) && 302 ntb_transport_link_query(q->qp)) 303 netif_wake_subqueue(ndev, q->qid); 304 } 305 } 306 307 static int ntb_netdev_open(struct net_device *ndev) 308 { 309 struct ntb_netdev *dev = netdev_priv(ndev); 310 struct ntb_netdev_queue *queue; 311 unsigned int q; 312 int rc = 0; 313 314 /* Add some empty rx bufs for each queue */ 315 for (q = 0; q < dev->num_queues; q++) { 316 queue = &dev->queues[q]; 317 318 rc = ntb_netdev_queue_rx_fill(ndev, queue); 319 if (rc) 320 goto err; 321 322 timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0); 323 } 324 325 netif_carrier_off(ndev); 326 netif_tx_stop_all_queues(ndev); 327 328 for (q = 0; q < dev->num_queues; q++) 329 ntb_transport_link_up(dev->queues[q].qp); 330 331 return 0; 332 333 err: 334 for (q = 0; q < dev->num_queues; q++) { 335 queue = &dev->queues[q]; 336 ntb_netdev_queue_rx_drain(queue); 337 } 338 return rc; 339 } 340 341 static int ntb_netdev_close(struct net_device *ndev) 342 { 343 struct ntb_netdev *dev = netdev_priv(ndev); 344 struct ntb_netdev_queue *queue; 345 unsigned int q; 346 347 netif_tx_stop_all_queues(ndev); 348 netif_carrier_off(ndev); 349 350 for (q = 0; q < dev->num_queues; q++) { 351 queue = &dev->queues[q]; 352 353 ntb_transport_link_down(queue->qp); 354 ntb_netdev_queue_rx_drain(queue); 355 timer_delete_sync(&queue->tx_timer); 356 } 357 358 return 0; 359 } 360 361 static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu) 362 { 363 struct ntb_netdev *dev = netdev_priv(ndev); 364 struct ntb_netdev_queue *queue; 365 struct sk_buff *skb; 366 unsigned int q, i; 367 int len, rc = 0; 368 369 if (new_mtu > ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN) 370 return -EINVAL; 371 372 if (!netif_running(ndev)) { 373 WRITE_ONCE(ndev->mtu, new_mtu); 374 return 0; 375 } 376 377 /* Bring down the link and dispose of posted rx entries */ 378 for (q = 0; q < dev->num_queues; q++) 379 ntb_transport_link_down(dev->queues[q].qp); 380 381 if (ndev->mtu < new_mtu) { 382 for (q = 0; q < dev->num_queues; q++) { 383 queue = &dev->queues[q]; 384 385 for (i = 0; 386 (skb = ntb_transport_rx_remove(queue->qp, &len)); 387 i++) 388 dev_kfree_skb(skb); 389 390 for (; i; i--) { 391 skb = netdev_alloc_skb(ndev, 392 new_mtu + ETH_HLEN); 393 if (!skb) { 394 rc = -ENOMEM; 395 goto err; 396 } 397 398 rc = ntb_transport_rx_enqueue(queue->qp, skb, 399 skb->data, 400 new_mtu + 401 ETH_HLEN); 402 if (rc) { 403 dev_kfree_skb(skb); 404 goto err; 405 } 406 } 407 } 408 } 409 410 WRITE_ONCE(ndev->mtu, new_mtu); 411 412 for (q = 0; q < dev->num_queues; q++) 413 ntb_transport_link_up(dev->queues[q].qp); 414 415 return 0; 416 417 err: 418 for (q = 0; q < dev->num_queues; q++) { 419 struct ntb_netdev_queue *queue = &dev->queues[q]; 420 421 ntb_transport_link_down(queue->qp); 422 423 ntb_netdev_queue_rx_drain(queue); 424 } 425 426 netdev_err(ndev, "Error changing MTU, device inoperable\n"); 427 return rc; 428 } 429 430 static const struct net_device_ops ntb_netdev_ops = { 431 .ndo_open = ntb_netdev_open, 432 .ndo_stop = ntb_netdev_close, 433 .ndo_start_xmit = ntb_netdev_start_xmit, 434 .ndo_change_mtu = ntb_netdev_change_mtu, 435 .ndo_set_mac_address = eth_mac_addr, 436 }; 437 438 static void ntb_get_drvinfo(struct net_device *ndev, 439 struct ethtool_drvinfo *info) 440 { 441 struct ntb_netdev *dev = netdev_priv(ndev); 442 443 strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 444 strscpy(info->version, NTB_NETDEV_VER, sizeof(info->version)); 445 strscpy(info->bus_info, pci_name(dev->pdev), sizeof(info->bus_info)); 446 } 447 448 static int ntb_get_link_ksettings(struct net_device *dev, 449 struct ethtool_link_ksettings *cmd) 450 { 451 ethtool_link_ksettings_zero_link_mode(cmd, supported); 452 ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane); 453 ethtool_link_ksettings_zero_link_mode(cmd, advertising); 454 ethtool_link_ksettings_add_link_mode(cmd, advertising, Backplane); 455 456 cmd->base.speed = SPEED_UNKNOWN; 457 cmd->base.duplex = DUPLEX_FULL; 458 cmd->base.port = PORT_OTHER; 459 cmd->base.phy_address = 0; 460 cmd->base.autoneg = AUTONEG_ENABLE; 461 462 return 0; 463 } 464 465 static void ntb_get_channels(struct net_device *ndev, 466 struct ethtool_channels *channels) 467 { 468 struct ntb_netdev *dev = netdev_priv(ndev); 469 470 channels->combined_count = dev->num_queues; 471 channels->max_combined = ndev->num_tx_queues; 472 } 473 474 static int ntb_inc_channels(struct net_device *ndev, 475 unsigned int old, unsigned int new) 476 { 477 struct ntb_netdev *dev = netdev_priv(ndev); 478 bool running = netif_running(ndev); 479 struct ntb_netdev_queue *queue; 480 unsigned int q, created; 481 int rc; 482 483 created = old; 484 for (q = old; q < new; q++) { 485 queue = &dev->queues[q]; 486 487 queue->ntdev = dev; 488 queue->qid = q; 489 queue->qp = ntb_transport_create_queue(queue, dev->client_dev, 490 &ntb_netdev_handlers); 491 if (!queue->qp) { 492 rc = -ENOSPC; 493 goto err_new; 494 } 495 created++; 496 497 if (!running) 498 continue; 499 500 timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0); 501 502 rc = ntb_netdev_queue_rx_fill(ndev, queue); 503 if (rc) 504 goto err_new; 505 506 /* 507 * Carrier may already be on due to other QPs. Keep the new 508 * subqueue stopped until we get a Link Up event for this QP. 509 */ 510 netif_stop_subqueue(ndev, q); 511 } 512 513 rc = netif_set_real_num_queues(ndev, new, new); 514 if (rc) 515 goto err_new; 516 517 dev->num_queues = new; 518 519 if (running) 520 for (q = old; q < new; q++) 521 ntb_transport_link_up(dev->queues[q].qp); 522 523 return 0; 524 525 err_new: 526 if (running) { 527 unsigned int rollback = created; 528 529 while (rollback-- > old) { 530 queue = &dev->queues[rollback]; 531 ntb_transport_link_down(queue->qp); 532 ntb_netdev_queue_rx_drain(queue); 533 timer_delete_sync(&queue->tx_timer); 534 } 535 } 536 while (created-- > old) { 537 queue = &dev->queues[created]; 538 ntb_transport_free_queue(queue->qp); 539 queue->qp = NULL; 540 } 541 return rc; 542 } 543 544 static int ntb_dec_channels(struct net_device *ndev, 545 unsigned int old, unsigned int new) 546 { 547 struct ntb_netdev *dev = netdev_priv(ndev); 548 bool running = netif_running(ndev); 549 struct ntb_netdev_queue *queue; 550 unsigned int q; 551 int rc; 552 553 if (running) 554 for (q = new; q < old; q++) 555 netif_stop_subqueue(ndev, q); 556 557 rc = netif_set_real_num_queues(ndev, new, new); 558 if (rc) 559 goto err; 560 561 /* Publish new queue count before invalidating QP pointers */ 562 dev->num_queues = new; 563 564 for (q = new; q < old; q++) { 565 queue = &dev->queues[q]; 566 567 if (running) { 568 ntb_transport_link_down(queue->qp); 569 ntb_netdev_queue_rx_drain(queue); 570 timer_delete_sync(&queue->tx_timer); 571 } 572 573 ntb_transport_free_queue(queue->qp); 574 queue->qp = NULL; 575 } 576 577 /* 578 * It might be the case that the removed queues are the only queues that 579 * were up, so see if the global carrier needs to change. 580 */ 581 ntb_netdev_update_carrier(dev); 582 return 0; 583 584 err: 585 if (running) { 586 for (q = new; q < old; q++) 587 netif_wake_subqueue(ndev, q); 588 } 589 return rc; 590 } 591 592 static int ntb_set_channels(struct net_device *ndev, 593 struct ethtool_channels *channels) 594 { 595 struct ntb_netdev *dev = netdev_priv(ndev); 596 unsigned int new = channels->combined_count; 597 unsigned int old = dev->num_queues; 598 599 if (new == old) 600 return 0; 601 602 if (new < old) 603 return ntb_dec_channels(ndev, old, new); 604 else 605 return ntb_inc_channels(ndev, old, new); 606 } 607 608 static const struct ethtool_ops ntb_ethtool_ops = { 609 .get_drvinfo = ntb_get_drvinfo, 610 .get_link = ethtool_op_get_link, 611 .get_link_ksettings = ntb_get_link_ksettings, 612 .get_channels = ntb_get_channels, 613 .set_channels = ntb_set_channels, 614 }; 615 616 static int ntb_netdev_probe(struct device *client_dev) 617 { 618 struct ntb_dev *ntb; 619 struct net_device *ndev; 620 struct pci_dev *pdev; 621 struct ntb_netdev *dev; 622 unsigned int q; 623 int rc; 624 625 ntb = dev_ntb(client_dev->parent); 626 pdev = ntb->pdev; 627 if (!pdev) 628 return -ENODEV; 629 630 ndev = alloc_etherdev_mq(sizeof(*dev), NTB_NETDEV_MAX_QUEUES); 631 if (!ndev) 632 return -ENOMEM; 633 634 SET_NETDEV_DEV(ndev, client_dev); 635 636 dev = netdev_priv(ndev); 637 dev->ndev = ndev; 638 dev->pdev = pdev; 639 dev->client_dev = client_dev; 640 dev->num_queues = 0; 641 642 dev->queues = kzalloc_objs(*dev->queues, NTB_NETDEV_MAX_QUEUES, 643 GFP_KERNEL); 644 if (!dev->queues) { 645 rc = -ENOMEM; 646 goto err_free_netdev; 647 } 648 649 ndev->features = NETIF_F_HIGHDMA; 650 651 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 652 653 ndev->hw_features = ndev->features; 654 ndev->watchdog_timeo = msecs_to_jiffies(NTB_TX_TIMEOUT_MS); 655 656 eth_random_addr(ndev->perm_addr); 657 dev_addr_set(ndev, ndev->perm_addr); 658 659 ndev->netdev_ops = &ntb_netdev_ops; 660 ndev->ethtool_ops = &ntb_ethtool_ops; 661 662 ndev->min_mtu = 0; 663 ndev->max_mtu = ETH_MAX_MTU; 664 665 for (q = 0; q < NTB_NETDEV_DEFAULT_QUEUES; q++) { 666 struct ntb_netdev_queue *queue = &dev->queues[q]; 667 668 queue->ntdev = dev; 669 queue->qid = q; 670 queue->qp = ntb_transport_create_queue(queue, client_dev, 671 &ntb_netdev_handlers); 672 if (!queue->qp) 673 break; 674 675 dev->num_queues++; 676 } 677 678 if (!dev->num_queues) { 679 rc = -EIO; 680 goto err_free_queues; 681 } 682 683 rc = netif_set_real_num_queues(ndev, dev->num_queues, dev->num_queues); 684 if (rc) 685 goto err_free_qps; 686 687 ndev->mtu = ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN; 688 689 rc = register_netdev(ndev); 690 if (rc) 691 goto err_free_qps; 692 693 dev_set_drvdata(client_dev, ndev); 694 dev_info(&pdev->dev, "%s created with %u queue pairs\n", 695 ndev->name, dev->num_queues); 696 return 0; 697 698 err_free_qps: 699 for (q = 0; q < dev->num_queues; q++) 700 ntb_transport_free_queue(dev->queues[q].qp); 701 702 err_free_queues: 703 kfree(dev->queues); 704 705 err_free_netdev: 706 free_netdev(ndev); 707 return rc; 708 } 709 710 static void ntb_netdev_remove(struct device *client_dev) 711 { 712 struct net_device *ndev = dev_get_drvdata(client_dev); 713 struct ntb_netdev *dev = netdev_priv(ndev); 714 unsigned int q; 715 716 unregister_netdev(ndev); 717 for (q = 0; q < dev->num_queues; q++) 718 ntb_transport_free_queue(dev->queues[q].qp); 719 720 kfree(dev->queues); 721 free_netdev(ndev); 722 } 723 724 static struct ntb_transport_client ntb_netdev_client = { 725 .driver.name = KBUILD_MODNAME, 726 .driver.owner = THIS_MODULE, 727 .probe = ntb_netdev_probe, 728 .remove = ntb_netdev_remove, 729 }; 730 731 static int __init ntb_netdev_init_module(void) 732 { 733 int rc; 734 735 rc = ntb_transport_register_client_dev(KBUILD_MODNAME); 736 if (rc) 737 return rc; 738 739 rc = ntb_transport_register_client(&ntb_netdev_client); 740 if (rc) { 741 ntb_transport_unregister_client_dev(KBUILD_MODNAME); 742 return rc; 743 } 744 745 return 0; 746 } 747 late_initcall(ntb_netdev_init_module); 748 749 static void __exit ntb_netdev_exit_module(void) 750 { 751 ntb_transport_unregister_client(&ntb_netdev_client); 752 ntb_transport_unregister_client_dev(KBUILD_MODNAME); 753 } 754 module_exit(ntb_netdev_exit_module); 755