1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright(c) 2012 Intel Corporation. All rights reserved. 17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of Intel Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * PCIe NTB Transport Linux driver 46 * 47 * Contact Information: 48 * Jon Mason <jon.mason@intel.com> 49 */ 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/dmaengine.h> 53 #include <linux/dma-mapping.h> 54 #include <linux/errno.h> 55 #include <linux/export.h> 56 #include <linux/interrupt.h> 57 #include <linux/kthread.h> 58 #include <linux/module.h> 59 #include <linux/pci.h> 60 #include <linux/slab.h> 61 #include <linux/seq_file.h> 62 #include <linux/types.h> 63 #include <linux/uaccess.h> 64 #include <linux/mutex.h> 65 #include <linux/wait.h> 66 #include "linux/ntb.h" 67 #include "linux/ntb_transport.h" 68 69 #define NTB_TRANSPORT_VERSION 4 70 #define NTB_TRANSPORT_VER "4" 71 #define NTB_TRANSPORT_NAME "ntb_transport" 72 #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" 73 #define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2) 74 75 MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); 76 MODULE_VERSION(NTB_TRANSPORT_VER); 77 MODULE_LICENSE("Dual BSD/GPL"); 78 MODULE_AUTHOR("Intel Corporation"); 79 80 static unsigned long max_mw_size; 81 module_param(max_mw_size, ulong, 0644); 82 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 83 84 static unsigned int transport_mtu = 0x10000; 85 module_param(transport_mtu, uint, 0644); 86 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 87 88 static unsigned char max_num_clients; 89 module_param(max_num_clients, byte, 0644); 90 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 91 92 static unsigned int copy_bytes = 1024; 93 module_param(copy_bytes, uint, 0644); 94 MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 95 96 static bool use_dma; 97 module_param(use_dma, bool, 0644); 98 MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); 99 100 static bool use_msi; 101 #ifdef CONFIG_NTB_MSI 102 module_param(use_msi, bool, 0644); 103 MODULE_PARM_DESC(use_msi, "Use MSI interrupts instead of doorbells"); 104 #endif 105 106 static bool tx_memcpy_offload; 107 module_param(tx_memcpy_offload, bool, 0644); 108 MODULE_PARM_DESC(tx_memcpy_offload, "Offload TX memcpy_toio() to a kernel thread"); 109 110 static struct dentry *nt_debugfs_dir; 111 112 /* Only two-ports NTB devices are supported */ 113 #define PIDX NTB_DEF_PEER_IDX 114 115 struct ntb_queue_entry { 116 /* ntb_queue list reference */ 117 struct list_head entry; 118 /* pointers to data to be transferred */ 119 void *cb_data; 120 void *buf; 121 unsigned int len; 122 unsigned int flags; 123 int errors; 124 unsigned int tx_index; 125 unsigned int rx_index; 126 127 struct ntb_transport_qp *qp; 128 union { 129 struct ntb_payload_header __iomem *tx_hdr; 130 struct ntb_payload_header *rx_hdr; 131 }; 132 }; 133 134 struct ntb_rx_info { 135 unsigned int entry; 136 }; 137 138 struct ntb_transport_qp { 139 struct ntb_transport_ctx *transport; 140 struct ntb_dev *ndev; 141 void *cb_data; 142 struct dma_chan *tx_dma_chan; 143 struct dma_chan *rx_dma_chan; 144 145 bool client_ready; 146 bool link_is_up; 147 bool active; 148 149 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 150 u64 qp_bit; 151 152 struct ntb_rx_info __iomem *rx_info; 153 struct ntb_rx_info *remote_rx_info; 154 155 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 156 void *data, int len); 157 struct list_head tx_free_q; 158 struct list_head tx_offl_q; 159 spinlock_t ntb_tx_free_q_lock; 160 spinlock_t ntb_tx_offl_q_lock; 161 void __iomem *tx_mw; 162 phys_addr_t tx_mw_phys; 163 size_t tx_mw_size; 164 dma_addr_t tx_mw_dma_addr; 165 unsigned int tx_index; 166 unsigned int tx_max_entry; 167 unsigned int tx_max_frame; 168 169 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 170 void *data, int len); 171 struct list_head rx_post_q; 172 struct list_head rx_pend_q; 173 struct list_head rx_free_q; 174 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 175 spinlock_t ntb_rx_q_lock; 176 void *rx_buff; 177 unsigned int rx_index; 178 unsigned int rx_max_entry; 179 unsigned int rx_max_frame; 180 unsigned int rx_alloc_entry; 181 dma_cookie_t last_cookie; 182 struct tasklet_struct rxc_db_work; 183 184 void (*event_handler)(void *data, int status); 185 struct delayed_work link_work; 186 struct work_struct link_cleanup; 187 188 struct dentry *debugfs_dir; 189 struct dentry *debugfs_stats; 190 191 /* Stats */ 192 u64 rx_bytes; 193 u64 rx_pkts; 194 u64 rx_ring_empty; 195 u64 rx_err_no_buf; 196 u64 rx_err_oflow; 197 u64 rx_err_ver; 198 u64 rx_memcpy; 199 u64 rx_async; 200 u64 tx_bytes; 201 u64 tx_pkts; 202 u64 tx_ring_full; 203 u64 tx_err_no_buf; 204 u64 tx_memcpy; 205 u64 tx_async; 206 207 bool use_msi; 208 int msi_irq; 209 struct ntb_msi_desc msi_desc; 210 struct ntb_msi_desc peer_msi_desc; 211 212 struct task_struct *tx_offload_thread; 213 wait_queue_head_t tx_offload_wq; 214 }; 215 216 struct ntb_transport_mw { 217 phys_addr_t phys_addr; 218 resource_size_t phys_size; 219 void __iomem *vbase; 220 size_t xlat_size; 221 size_t buff_size; 222 size_t alloc_size; 223 void *alloc_addr; 224 void *virt_addr; 225 dma_addr_t dma_addr; 226 dma_addr_t original_dma_addr; 227 }; 228 229 struct ntb_transport_client_dev { 230 struct list_head entry; 231 struct ntb_transport_ctx *nt; 232 struct device dev; 233 }; 234 235 struct ntb_transport_ctx { 236 struct list_head entry; 237 struct list_head client_devs; 238 239 struct ntb_dev *ndev; 240 241 struct ntb_transport_mw *mw_vec; 242 struct ntb_transport_qp *qp_vec; 243 unsigned int mw_count; 244 unsigned int qp_count; 245 u64 qp_bitmap; 246 u64 qp_bitmap_free; 247 248 bool use_msi; 249 unsigned int msi_spad_offset; 250 u64 msi_db_mask; 251 252 bool link_is_up; 253 struct delayed_work link_work; 254 struct work_struct link_cleanup; 255 256 struct dentry *debugfs_node_dir; 257 258 /* Make sure workq of link event be executed serially */ 259 struct mutex link_event_lock; 260 }; 261 262 enum { 263 DESC_DONE_FLAG = BIT(0), 264 LINK_DOWN_FLAG = BIT(1), 265 }; 266 267 struct ntb_payload_header { 268 unsigned int ver; 269 unsigned int len; 270 unsigned int flags; 271 }; 272 273 enum { 274 VERSION = 0, 275 QP_LINKS, 276 NUM_QPS, 277 NUM_MWS, 278 MW0_SZ_HIGH, 279 MW0_SZ_LOW, 280 }; 281 282 #define dev_client_dev(__dev) \ 283 container_of((__dev), struct ntb_transport_client_dev, dev) 284 285 #define drv_client(__drv) \ 286 container_of((__drv), struct ntb_transport_client, driver) 287 288 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 289 #define NTB_QP_DEF_NUM_ENTRIES 100 290 #define NTB_LINK_DOWN_TIMEOUT 10 291 292 static void ntb_transport_rxc_db(unsigned long data); 293 static const struct ntb_ctx_ops ntb_transport_ops; 294 static struct ntb_client ntb_transport_client; 295 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 296 struct ntb_queue_entry *entry); 297 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset); 298 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset); 299 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset); 300 static int ntb_tx_memcpy_kthread(void *data); 301 302 303 static inline bool ntb_tx_offload_enabled(struct ntb_transport_qp *qp) 304 { 305 return tx_memcpy_offload && qp && qp->tx_offload_thread; 306 } 307 308 static int ntb_transport_bus_match(struct device *dev, 309 const struct device_driver *drv) 310 { 311 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 312 } 313 314 static int ntb_transport_bus_probe(struct device *dev) 315 { 316 const struct ntb_transport_client *client; 317 int rc; 318 319 get_device(dev); 320 321 client = drv_client(dev->driver); 322 rc = client->probe(dev); 323 if (rc) 324 put_device(dev); 325 326 return rc; 327 } 328 329 static void ntb_transport_bus_remove(struct device *dev) 330 { 331 const struct ntb_transport_client *client; 332 333 client = drv_client(dev->driver); 334 client->remove(dev); 335 336 put_device(dev); 337 } 338 339 static const struct bus_type ntb_transport_bus = { 340 .name = "ntb_transport", 341 .match = ntb_transport_bus_match, 342 .probe = ntb_transport_bus_probe, 343 .remove = ntb_transport_bus_remove, 344 }; 345 346 static LIST_HEAD(ntb_transport_list); 347 348 static int ntb_bus_init(struct ntb_transport_ctx *nt) 349 { 350 list_add_tail(&nt->entry, &ntb_transport_list); 351 return 0; 352 } 353 354 static void ntb_bus_remove(struct ntb_transport_ctx *nt) 355 { 356 struct ntb_transport_client_dev *client_dev, *cd; 357 358 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 359 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 360 dev_name(&client_dev->dev)); 361 list_del(&client_dev->entry); 362 device_unregister(&client_dev->dev); 363 } 364 365 list_del(&nt->entry); 366 } 367 368 static void ntb_transport_client_release(struct device *dev) 369 { 370 struct ntb_transport_client_dev *client_dev; 371 372 client_dev = dev_client_dev(dev); 373 kfree(client_dev); 374 } 375 376 /** 377 * ntb_transport_unregister_client_dev - Unregister NTB client device 378 * @device_name: Name of NTB client device 379 * 380 * Unregister an NTB client device with the NTB transport layer 381 */ 382 void ntb_transport_unregister_client_dev(char *device_name) 383 { 384 struct ntb_transport_client_dev *client, *cd; 385 struct ntb_transport_ctx *nt; 386 387 list_for_each_entry(nt, &ntb_transport_list, entry) 388 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 389 if (!strncmp(dev_name(&client->dev), device_name, 390 strlen(device_name))) { 391 list_del(&client->entry); 392 device_unregister(&client->dev); 393 } 394 } 395 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); 396 397 /** 398 * ntb_transport_register_client_dev - Register NTB client device 399 * @device_name: Name of NTB client device 400 * 401 * Register an NTB client device with the NTB transport layer 402 * 403 * Returns: %0 on success or -errno code on error 404 */ 405 int ntb_transport_register_client_dev(char *device_name) 406 { 407 struct ntb_transport_client_dev *client_dev; 408 struct ntb_transport_ctx *nt; 409 int node; 410 int rc, i = 0; 411 412 if (list_empty(&ntb_transport_list)) 413 return -ENODEV; 414 415 list_for_each_entry(nt, &ntb_transport_list, entry) { 416 struct device *dev; 417 418 node = dev_to_node(&nt->ndev->dev); 419 420 client_dev = kzalloc_node(sizeof(*client_dev), 421 GFP_KERNEL, node); 422 if (!client_dev) { 423 rc = -ENOMEM; 424 goto err; 425 } 426 427 dev = &client_dev->dev; 428 429 /* setup and register client devices */ 430 dev_set_name(dev, "%s%d", device_name, i); 431 dev->bus = &ntb_transport_bus; 432 dev->release = ntb_transport_client_release; 433 dev->parent = &nt->ndev->dev; 434 435 rc = device_register(dev); 436 if (rc) { 437 put_device(dev); 438 goto err; 439 } 440 441 list_add_tail(&client_dev->entry, &nt->client_devs); 442 i++; 443 } 444 445 return 0; 446 447 err: 448 ntb_transport_unregister_client_dev(device_name); 449 450 return rc; 451 } 452 EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); 453 454 /** 455 * ntb_transport_register_client - Register NTB client driver 456 * @drv: NTB client driver to be registered 457 * 458 * Register an NTB client driver with the NTB transport layer 459 * 460 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 461 */ 462 int ntb_transport_register_client(struct ntb_transport_client *drv) 463 { 464 drv->driver.bus = &ntb_transport_bus; 465 466 if (list_empty(&ntb_transport_list)) 467 return -ENODEV; 468 469 return driver_register(&drv->driver); 470 } 471 EXPORT_SYMBOL_GPL(ntb_transport_register_client); 472 473 /** 474 * ntb_transport_unregister_client - Unregister NTB client driver 475 * @drv: NTB client driver to be unregistered 476 * 477 * Unregister an NTB client driver with the NTB transport layer 478 * 479 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 480 */ 481 void ntb_transport_unregister_client(struct ntb_transport_client *drv) 482 { 483 driver_unregister(&drv->driver); 484 } 485 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); 486 487 static int ntb_qp_debugfs_stats_show(struct seq_file *s, void *v) 488 { 489 struct ntb_transport_qp *qp = s->private; 490 491 if (!qp || !qp->link_is_up) 492 return 0; 493 494 seq_puts(s, "\nNTB QP stats:\n\n"); 495 496 seq_printf(s, "rx_bytes - \t%llu\n", qp->rx_bytes); 497 seq_printf(s, "rx_pkts - \t%llu\n", qp->rx_pkts); 498 seq_printf(s, "rx_memcpy - \t%llu\n", qp->rx_memcpy); 499 seq_printf(s, "rx_async - \t%llu\n", qp->rx_async); 500 seq_printf(s, "rx_ring_empty - %llu\n", qp->rx_ring_empty); 501 seq_printf(s, "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 502 seq_printf(s, "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 503 seq_printf(s, "rx_err_ver - \t%llu\n", qp->rx_err_ver); 504 seq_printf(s, "rx_buff - \t0x%p\n", qp->rx_buff); 505 seq_printf(s, "rx_index - \t%u\n", qp->rx_index); 506 seq_printf(s, "rx_max_entry - \t%u\n", qp->rx_max_entry); 507 seq_printf(s, "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry); 508 509 seq_printf(s, "tx_bytes - \t%llu\n", qp->tx_bytes); 510 seq_printf(s, "tx_pkts - \t%llu\n", qp->tx_pkts); 511 seq_printf(s, "tx_memcpy - \t%llu\n", qp->tx_memcpy); 512 seq_printf(s, "tx_async - \t%llu\n", qp->tx_async); 513 seq_printf(s, "tx_ring_full - \t%llu\n", qp->tx_ring_full); 514 seq_printf(s, "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 515 seq_printf(s, "tx_mw - \t0x%p\n", qp->tx_mw); 516 seq_printf(s, "tx_index (H) - \t%u\n", qp->tx_index); 517 seq_printf(s, "RRI (T) - \t%u\n", qp->remote_rx_info->entry); 518 seq_printf(s, "tx_max_entry - \t%u\n", qp->tx_max_entry); 519 seq_printf(s, "free tx - \t%u\n", ntb_transport_tx_free_entry(qp)); 520 seq_putc(s, '\n'); 521 522 seq_printf(s, "Using TX DMA - \t%s\n", qp->tx_dma_chan ? "Yes" : "No"); 523 seq_printf(s, "Using RX DMA - \t%s\n", qp->rx_dma_chan ? "Yes" : "No"); 524 seq_printf(s, "QP Link - \t%s\n", qp->link_is_up ? "Up" : "Down"); 525 seq_putc(s, '\n'); 526 527 return 0; 528 } 529 DEFINE_SHOW_ATTRIBUTE(ntb_qp_debugfs_stats); 530 531 static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 532 struct list_head *list) 533 { 534 unsigned long flags; 535 536 spin_lock_irqsave(lock, flags); 537 list_add_tail(entry, list); 538 spin_unlock_irqrestore(lock, flags); 539 } 540 541 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 542 struct list_head *list) 543 { 544 struct ntb_queue_entry *entry; 545 unsigned long flags; 546 547 spin_lock_irqsave(lock, flags); 548 if (list_empty(list)) { 549 entry = NULL; 550 goto out; 551 } 552 entry = list_first_entry(list, struct ntb_queue_entry, entry); 553 list_del(&entry->entry); 554 555 out: 556 spin_unlock_irqrestore(lock, flags); 557 558 return entry; 559 } 560 561 static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock, 562 struct list_head *list, 563 struct list_head *to_list) 564 { 565 struct ntb_queue_entry *entry; 566 unsigned long flags; 567 568 spin_lock_irqsave(lock, flags); 569 570 if (list_empty(list)) { 571 entry = NULL; 572 } else { 573 entry = list_first_entry(list, struct ntb_queue_entry, entry); 574 list_move_tail(&entry->entry, to_list); 575 } 576 577 spin_unlock_irqrestore(lock, flags); 578 579 return entry; 580 } 581 582 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 583 unsigned int qp_num) 584 { 585 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 586 struct ntb_transport_mw *mw; 587 struct ntb_dev *ndev = nt->ndev; 588 struct ntb_queue_entry *entry; 589 unsigned int rx_size, num_qps_mw; 590 unsigned int mw_num, mw_count, qp_count; 591 unsigned int i; 592 int node; 593 594 mw_count = nt->mw_count; 595 qp_count = nt->qp_count; 596 597 mw_num = QP_TO_MW(nt, qp_num); 598 mw = &nt->mw_vec[mw_num]; 599 600 if (!mw->virt_addr) 601 return -ENOMEM; 602 603 if (mw_num < qp_count % mw_count) 604 num_qps_mw = qp_count / mw_count + 1; 605 else 606 num_qps_mw = qp_count / mw_count; 607 608 rx_size = (unsigned int)mw->xlat_size / num_qps_mw; 609 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 610 rx_size -= sizeof(struct ntb_rx_info); 611 612 qp->remote_rx_info = qp->rx_buff + rx_size; 613 614 /* Due to housekeeping, there must be atleast 2 buffs */ 615 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 616 qp->rx_max_entry = rx_size / qp->rx_max_frame; 617 qp->rx_index = 0; 618 619 /* 620 * Checking to see if we have more entries than the default. 621 * We should add additional entries if that is the case so we 622 * can be in sync with the transport frames. 623 */ 624 node = dev_to_node(&ndev->dev); 625 for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { 626 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 627 if (!entry) 628 return -ENOMEM; 629 630 entry->qp = qp; 631 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 632 &qp->rx_free_q); 633 qp->rx_alloc_entry++; 634 } 635 636 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 637 638 /* setup the hdr offsets with 0's */ 639 for (i = 0; i < qp->rx_max_entry; i++) { 640 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - 641 sizeof(struct ntb_payload_header)); 642 memset(offset, 0, sizeof(struct ntb_payload_header)); 643 } 644 645 qp->rx_pkts = 0; 646 qp->tx_pkts = 0; 647 qp->tx_index = 0; 648 649 return 0; 650 } 651 652 static irqreturn_t ntb_transport_isr(int irq, void *dev) 653 { 654 struct ntb_transport_qp *qp = dev; 655 656 tasklet_schedule(&qp->rxc_db_work); 657 658 return IRQ_HANDLED; 659 } 660 661 static void ntb_transport_setup_qp_peer_msi(struct ntb_transport_ctx *nt, 662 unsigned int qp_num) 663 { 664 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 665 int spad = qp_num * 2 + nt->msi_spad_offset; 666 667 if (!nt->use_msi) 668 return; 669 670 if (spad >= ntb_spad_count(nt->ndev)) 671 return; 672 673 qp->peer_msi_desc.addr_offset = 674 ntb_peer_spad_read(qp->ndev, PIDX, spad); 675 qp->peer_msi_desc.data = 676 ntb_peer_spad_read(qp->ndev, PIDX, spad + 1); 677 678 dev_dbg(&qp->ndev->pdev->dev, "QP%d Peer MSI addr=%x data=%x\n", 679 qp_num, qp->peer_msi_desc.addr_offset, qp->peer_msi_desc.data); 680 681 if (qp->peer_msi_desc.addr_offset) { 682 qp->use_msi = true; 683 dev_info(&qp->ndev->pdev->dev, 684 "Using MSI interrupts for QP%d\n", qp_num); 685 } 686 } 687 688 static void ntb_transport_setup_qp_msi(struct ntb_transport_ctx *nt, 689 unsigned int qp_num) 690 { 691 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 692 int spad = qp_num * 2 + nt->msi_spad_offset; 693 int rc; 694 695 if (!nt->use_msi) 696 return; 697 698 if (spad >= ntb_spad_count(nt->ndev)) { 699 dev_warn_once(&qp->ndev->pdev->dev, 700 "Not enough SPADS to use MSI interrupts\n"); 701 return; 702 } 703 704 ntb_spad_write(qp->ndev, spad, 0); 705 ntb_spad_write(qp->ndev, spad + 1, 0); 706 707 if (!qp->msi_irq) { 708 qp->msi_irq = ntbm_msi_request_irq(qp->ndev, ntb_transport_isr, 709 KBUILD_MODNAME, qp, 710 &qp->msi_desc); 711 if (qp->msi_irq < 0) { 712 dev_warn(&qp->ndev->pdev->dev, 713 "Unable to allocate MSI interrupt for qp%d\n", 714 qp_num); 715 return; 716 } 717 } 718 719 rc = ntb_spad_write(qp->ndev, spad, qp->msi_desc.addr_offset); 720 if (rc) 721 goto err_free_interrupt; 722 723 rc = ntb_spad_write(qp->ndev, spad + 1, qp->msi_desc.data); 724 if (rc) 725 goto err_free_interrupt; 726 727 dev_dbg(&qp->ndev->pdev->dev, "QP%d MSI %d addr=%x data=%x\n", 728 qp_num, qp->msi_irq, qp->msi_desc.addr_offset, 729 qp->msi_desc.data); 730 731 return; 732 733 err_free_interrupt: 734 devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp); 735 } 736 737 static void ntb_transport_msi_peer_desc_changed(struct ntb_transport_ctx *nt) 738 { 739 int i; 740 741 dev_dbg(&nt->ndev->pdev->dev, "Peer MSI descriptors changed"); 742 743 for (i = 0; i < nt->qp_count; i++) 744 ntb_transport_setup_qp_peer_msi(nt, i); 745 } 746 747 static void ntb_transport_msi_desc_changed(void *data) 748 { 749 struct ntb_transport_ctx *nt = data; 750 int i; 751 752 dev_dbg(&nt->ndev->pdev->dev, "MSI descriptors changed"); 753 754 for (i = 0; i < nt->qp_count; i++) 755 ntb_transport_setup_qp_msi(nt, i); 756 757 ntb_peer_db_set(nt->ndev, nt->msi_db_mask); 758 } 759 760 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 761 { 762 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 763 struct device *dma_dev = ntb_get_dma_dev(nt->ndev); 764 765 if (!mw->virt_addr) 766 return; 767 768 ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); 769 dma_free_attrs(dma_dev, mw->alloc_size, mw->alloc_addr, 770 mw->original_dma_addr, DMA_ATTR_FORCE_CONTIGUOUS); 771 mw->xlat_size = 0; 772 mw->buff_size = 0; 773 mw->alloc_size = 0; 774 mw->alloc_addr = NULL; 775 mw->virt_addr = NULL; 776 } 777 778 static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, 779 struct device *ntb_dev, size_t align) 780 { 781 dma_addr_t dma_addr; 782 void *alloc_addr, *virt_addr; 783 int rc; 784 785 /* 786 * The buffer here is allocated against the NTB device. The reason to 787 * use dma_alloc_*() call is to allocate a large IOVA contiguous buffer 788 * backing the NTB BAR for the remote host to write to. During receive 789 * processing, the data is being copied out of the receive buffer to 790 * the kernel skbuff. When a DMA device is being used, dma_map_page() 791 * is called on the kvaddr of the receive buffer (from dma_alloc_*()) 792 * and remapped against the DMA device. It appears to be a double 793 * DMA mapping of buffers, but first is mapped to the NTB device and 794 * second is to the DMA device. DMA_ATTR_FORCE_CONTIGUOUS is necessary 795 * in order for the later dma_map_page() to not fail. 796 */ 797 alloc_addr = dma_alloc_attrs(ntb_dev, mw->alloc_size, 798 &dma_addr, GFP_KERNEL, 799 DMA_ATTR_FORCE_CONTIGUOUS); 800 if (!alloc_addr) { 801 dev_err(ntb_dev, "Unable to alloc MW buff of size %zu\n", 802 mw->alloc_size); 803 return -ENOMEM; 804 } 805 virt_addr = alloc_addr; 806 mw->original_dma_addr = dma_addr; 807 808 /* 809 * we must ensure that the memory address allocated is BAR size 810 * aligned in order for the XLAT register to take the value. This 811 * is a requirement of the hardware. It is recommended to setup CMA 812 * for BAR sizes equal or greater than 4MB. 813 */ 814 if (!IS_ALIGNED(dma_addr, align)) { 815 if (mw->alloc_size > mw->buff_size) { 816 virt_addr = PTR_ALIGN(alloc_addr, align); 817 dma_addr = ALIGN(dma_addr, align); 818 } else { 819 rc = -ENOMEM; 820 goto err; 821 } 822 } 823 824 mw->alloc_addr = alloc_addr; 825 mw->virt_addr = virt_addr; 826 mw->dma_addr = dma_addr; 827 828 return 0; 829 830 err: 831 dma_free_attrs(ntb_dev, mw->alloc_size, alloc_addr, dma_addr, 832 DMA_ATTR_FORCE_CONTIGUOUS); 833 return rc; 834 } 835 836 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, 837 resource_size_t size) 838 { 839 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 840 struct device *dma_dev = ntb_get_dma_dev(nt->ndev); 841 size_t xlat_size, buff_size; 842 resource_size_t xlat_align; 843 resource_size_t xlat_align_size; 844 int rc; 845 846 if (!size) 847 return -EINVAL; 848 849 rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align, 850 &xlat_align_size, NULL); 851 if (rc) 852 return rc; 853 854 xlat_size = round_up(size, xlat_align_size); 855 buff_size = round_up(size, xlat_align); 856 857 /* No need to re-setup */ 858 if (mw->xlat_size == xlat_size) 859 return 0; 860 861 if (mw->buff_size) 862 ntb_free_mw(nt, num_mw); 863 864 /* Alloc memory for receiving data. Must be aligned */ 865 mw->xlat_size = xlat_size; 866 mw->buff_size = buff_size; 867 mw->alloc_size = buff_size; 868 869 rc = ntb_alloc_mw_buffer(mw, dma_dev, xlat_align); 870 if (rc) { 871 mw->alloc_size *= 2; 872 rc = ntb_alloc_mw_buffer(mw, dma_dev, xlat_align); 873 if (rc) { 874 dev_err(dma_dev, 875 "Unable to alloc aligned MW buff\n"); 876 mw->xlat_size = 0; 877 mw->buff_size = 0; 878 mw->alloc_size = 0; 879 return rc; 880 } 881 } 882 883 /* Notify HW the memory location of the receive buffer */ 884 rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr, 885 mw->xlat_size); 886 if (rc) { 887 dev_err(dma_dev, "Unable to set mw%d translation", num_mw); 888 ntb_free_mw(nt, num_mw); 889 return -EIO; 890 } 891 892 return 0; 893 } 894 895 static void ntb_qp_link_context_reset(struct ntb_transport_qp *qp) 896 { 897 qp->link_is_up = false; 898 qp->active = false; 899 900 qp->tx_index = 0; 901 qp->rx_index = 0; 902 qp->rx_bytes = 0; 903 qp->rx_pkts = 0; 904 qp->rx_ring_empty = 0; 905 qp->rx_err_no_buf = 0; 906 qp->rx_err_oflow = 0; 907 qp->rx_err_ver = 0; 908 qp->rx_memcpy = 0; 909 qp->rx_async = 0; 910 qp->tx_bytes = 0; 911 qp->tx_pkts = 0; 912 qp->tx_ring_full = 0; 913 qp->tx_err_no_buf = 0; 914 qp->tx_memcpy = 0; 915 qp->tx_async = 0; 916 } 917 918 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 919 { 920 ntb_qp_link_context_reset(qp); 921 if (qp->remote_rx_info) 922 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 923 } 924 925 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 926 { 927 struct ntb_transport_ctx *nt = qp->transport; 928 struct pci_dev *pdev = nt->ndev->pdev; 929 930 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); 931 932 cancel_delayed_work_sync(&qp->link_work); 933 ntb_qp_link_down_reset(qp); 934 935 if (qp->event_handler) 936 qp->event_handler(qp->cb_data, qp->link_is_up); 937 } 938 939 static void ntb_qp_link_cleanup_work(struct work_struct *work) 940 { 941 struct ntb_transport_qp *qp = container_of(work, 942 struct ntb_transport_qp, 943 link_cleanup); 944 struct ntb_transport_ctx *nt = qp->transport; 945 946 ntb_qp_link_cleanup(qp); 947 948 if (nt->link_is_up) 949 schedule_delayed_work(&qp->link_work, 950 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 951 } 952 953 static void ntb_qp_link_down(struct ntb_transport_qp *qp) 954 { 955 schedule_work(&qp->link_cleanup); 956 } 957 958 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 959 { 960 struct ntb_transport_qp *qp; 961 u64 qp_bitmap_alloc; 962 unsigned int i, count; 963 964 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 965 966 /* Pass along the info to any clients */ 967 for (i = 0; i < nt->qp_count; i++) 968 if (qp_bitmap_alloc & BIT_ULL(i)) { 969 qp = &nt->qp_vec[i]; 970 ntb_qp_link_cleanup(qp); 971 cancel_work_sync(&qp->link_cleanup); 972 cancel_delayed_work_sync(&qp->link_work); 973 } 974 975 if (!nt->link_is_up) 976 cancel_delayed_work_sync(&nt->link_work); 977 978 for (i = 0; i < nt->mw_count; i++) 979 ntb_free_mw(nt, i); 980 981 /* The scratchpad registers keep the values if the remote side 982 * goes down, blast them now to give them a sane value the next 983 * time they are accessed 984 */ 985 count = ntb_spad_count(nt->ndev); 986 for (i = 0; i < count; i++) 987 ntb_spad_write(nt->ndev, i, 0); 988 } 989 990 static void ntb_transport_link_cleanup_work(struct work_struct *work) 991 { 992 struct ntb_transport_ctx *nt = 993 container_of(work, struct ntb_transport_ctx, link_cleanup); 994 995 guard(mutex)(&nt->link_event_lock); 996 ntb_transport_link_cleanup(nt); 997 } 998 999 static void ntb_transport_event_callback(void *data) 1000 { 1001 struct ntb_transport_ctx *nt = data; 1002 1003 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) 1004 schedule_delayed_work(&nt->link_work, 0); 1005 else 1006 schedule_work(&nt->link_cleanup); 1007 } 1008 1009 static void ntb_transport_link_work(struct work_struct *work) 1010 { 1011 struct ntb_transport_ctx *nt = 1012 container_of(work, struct ntb_transport_ctx, link_work.work); 1013 struct ntb_dev *ndev = nt->ndev; 1014 struct pci_dev *pdev = ndev->pdev; 1015 resource_size_t size; 1016 u32 val; 1017 int rc = 0, i, spad; 1018 1019 guard(mutex)(&nt->link_event_lock); 1020 1021 /* send the local info, in the opposite order of the way we read it */ 1022 1023 if (nt->use_msi) { 1024 rc = ntb_msi_setup_mws(ndev); 1025 if (rc) { 1026 dev_warn(&pdev->dev, 1027 "Failed to register MSI memory window: %d\n", 1028 rc); 1029 nt->use_msi = false; 1030 } 1031 } 1032 1033 for (i = 0; i < nt->qp_count; i++) 1034 ntb_transport_setup_qp_msi(nt, i); 1035 1036 for (i = 0; i < nt->mw_count; i++) { 1037 size = nt->mw_vec[i].phys_size; 1038 1039 if (max_mw_size && size > max_mw_size) 1040 size = max_mw_size; 1041 1042 spad = MW0_SZ_HIGH + (i * 2); 1043 ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size)); 1044 1045 spad = MW0_SZ_LOW + (i * 2); 1046 ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size)); 1047 } 1048 1049 ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count); 1050 1051 ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count); 1052 1053 ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION); 1054 1055 /* Query the remote side for its info */ 1056 val = ntb_spad_read(ndev, VERSION); 1057 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 1058 if (val != NTB_TRANSPORT_VERSION) 1059 goto out; 1060 1061 val = ntb_spad_read(ndev, NUM_QPS); 1062 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 1063 if (val != nt->qp_count) 1064 goto out; 1065 1066 val = ntb_spad_read(ndev, NUM_MWS); 1067 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 1068 if (val != nt->mw_count) 1069 goto out; 1070 1071 for (i = 0; i < nt->mw_count; i++) { 1072 u64 val64; 1073 1074 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); 1075 val64 = (u64)val << 32; 1076 1077 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); 1078 val64 |= val; 1079 1080 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); 1081 1082 rc = ntb_set_mw(nt, i, val64); 1083 if (rc) 1084 goto out1; 1085 } 1086 1087 nt->link_is_up = true; 1088 1089 for (i = 0; i < nt->qp_count; i++) { 1090 struct ntb_transport_qp *qp = &nt->qp_vec[i]; 1091 1092 ntb_transport_setup_qp_mw(nt, i); 1093 ntb_transport_setup_qp_peer_msi(nt, i); 1094 1095 if (qp->client_ready) 1096 schedule_delayed_work(&qp->link_work, 0); 1097 } 1098 1099 return; 1100 1101 out1: 1102 for (i = 0; i < nt->mw_count; i++) 1103 ntb_free_mw(nt, i); 1104 1105 /* if there's an actual failure, we should just bail */ 1106 if (rc < 0) 1107 return; 1108 1109 out: 1110 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 1111 schedule_delayed_work(&nt->link_work, 1112 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 1113 } 1114 1115 static void ntb_qp_link_work(struct work_struct *work) 1116 { 1117 struct ntb_transport_qp *qp = container_of(work, 1118 struct ntb_transport_qp, 1119 link_work.work); 1120 struct pci_dev *pdev = qp->ndev->pdev; 1121 struct ntb_transport_ctx *nt = qp->transport; 1122 int val; 1123 1124 WARN_ON(!nt->link_is_up); 1125 1126 val = ntb_spad_read(nt->ndev, QP_LINKS); 1127 1128 ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num)); 1129 1130 /* query remote spad for qp ready bits */ 1131 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); 1132 1133 /* See if the remote side is up */ 1134 if (val & BIT(qp->qp_num)) { 1135 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 1136 qp->link_is_up = true; 1137 qp->active = true; 1138 1139 if (qp->event_handler) 1140 qp->event_handler(qp->cb_data, qp->link_is_up); 1141 1142 if (qp->active) 1143 tasklet_schedule(&qp->rxc_db_work); 1144 } else if (nt->link_is_up) 1145 schedule_delayed_work(&qp->link_work, 1146 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 1147 } 1148 1149 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, 1150 unsigned int qp_num) 1151 { 1152 struct ntb_transport_qp *qp; 1153 phys_addr_t mw_base; 1154 resource_size_t mw_size; 1155 unsigned int num_qps_mw, tx_size; 1156 unsigned int mw_num, mw_count, qp_count; 1157 u64 qp_offset; 1158 1159 mw_count = nt->mw_count; 1160 qp_count = nt->qp_count; 1161 1162 mw_num = QP_TO_MW(nt, qp_num); 1163 1164 qp = &nt->qp_vec[qp_num]; 1165 qp->qp_num = qp_num; 1166 qp->transport = nt; 1167 qp->ndev = nt->ndev; 1168 qp->client_ready = false; 1169 qp->event_handler = NULL; 1170 ntb_qp_link_context_reset(qp); 1171 1172 if (mw_num < qp_count % mw_count) 1173 num_qps_mw = qp_count / mw_count + 1; 1174 else 1175 num_qps_mw = qp_count / mw_count; 1176 1177 mw_base = nt->mw_vec[mw_num].phys_addr; 1178 mw_size = nt->mw_vec[mw_num].phys_size; 1179 1180 if (max_mw_size && mw_size > max_mw_size) 1181 mw_size = max_mw_size; 1182 1183 tx_size = (unsigned int)mw_size / num_qps_mw; 1184 qp_offset = tx_size * (qp_num / mw_count); 1185 1186 qp->tx_mw_size = tx_size; 1187 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; 1188 if (!qp->tx_mw) 1189 return -EINVAL; 1190 1191 qp->tx_mw_phys = mw_base + qp_offset; 1192 if (!qp->tx_mw_phys) 1193 return -EINVAL; 1194 1195 tx_size -= sizeof(struct ntb_rx_info); 1196 qp->rx_info = qp->tx_mw + tx_size; 1197 1198 /* Due to housekeeping, there must be atleast 2 buffs */ 1199 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 1200 qp->tx_max_entry = tx_size / qp->tx_max_frame; 1201 1202 if (nt->debugfs_node_dir) { 1203 char debugfs_name[8]; 1204 1205 snprintf(debugfs_name, sizeof(debugfs_name), "qp%d", qp_num); 1206 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 1207 nt->debugfs_node_dir); 1208 1209 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 1210 qp->debugfs_dir, qp, 1211 &ntb_qp_debugfs_stats_fops); 1212 } else { 1213 qp->debugfs_dir = NULL; 1214 qp->debugfs_stats = NULL; 1215 } 1216 1217 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 1218 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); 1219 1220 spin_lock_init(&qp->ntb_rx_q_lock); 1221 spin_lock_init(&qp->ntb_tx_free_q_lock); 1222 spin_lock_init(&qp->ntb_tx_offl_q_lock); 1223 1224 INIT_LIST_HEAD(&qp->rx_post_q); 1225 INIT_LIST_HEAD(&qp->rx_pend_q); 1226 INIT_LIST_HEAD(&qp->rx_free_q); 1227 INIT_LIST_HEAD(&qp->tx_free_q); 1228 INIT_LIST_HEAD(&qp->tx_offl_q); 1229 1230 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, 1231 (unsigned long)qp); 1232 1233 return 0; 1234 } 1235 1236 static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) 1237 { 1238 struct ntb_transport_ctx *nt; 1239 struct ntb_transport_mw *mw; 1240 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads; 1241 u64 qp_bitmap; 1242 int node; 1243 int rc, i; 1244 1245 mw_count = ntb_peer_mw_count(ndev); 1246 1247 if (!ndev->ops->mw_set_trans) { 1248 dev_err(&ndev->dev, "Inbound MW based NTB API is required\n"); 1249 return -EINVAL; 1250 } 1251 1252 if (ntb_db_is_unsafe(ndev)) 1253 dev_dbg(&ndev->dev, 1254 "doorbell is unsafe, proceed anyway...\n"); 1255 if (ntb_spad_is_unsafe(ndev)) 1256 dev_dbg(&ndev->dev, 1257 "scratchpad is unsafe, proceed anyway...\n"); 1258 1259 if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT) 1260 dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n"); 1261 1262 node = dev_to_node(&ndev->dev); 1263 1264 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); 1265 if (!nt) 1266 return -ENOMEM; 1267 1268 nt->ndev = ndev; 1269 1270 /* 1271 * If we are using MSI, and have at least one extra memory window, 1272 * we will reserve the last MW for the MSI window. 1273 */ 1274 if (use_msi && mw_count > 1) { 1275 rc = ntb_msi_init(ndev, ntb_transport_msi_desc_changed); 1276 if (!rc) { 1277 mw_count -= 1; 1278 nt->use_msi = true; 1279 } 1280 } 1281 1282 spad_count = ntb_spad_count(ndev); 1283 1284 /* Limit the MW's based on the availability of scratchpads */ 1285 1286 if (spad_count < NTB_TRANSPORT_MIN_SPADS) { 1287 nt->mw_count = 0; 1288 rc = -EINVAL; 1289 goto err; 1290 } 1291 1292 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2; 1293 nt->mw_count = min(mw_count, max_mw_count_for_spads); 1294 1295 nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH; 1296 1297 nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec), 1298 GFP_KERNEL, node); 1299 if (!nt->mw_vec) { 1300 rc = -ENOMEM; 1301 goto err; 1302 } 1303 1304 for (i = 0; i < mw_count; i++) { 1305 mw = &nt->mw_vec[i]; 1306 1307 rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr, 1308 &mw->phys_size); 1309 if (rc) 1310 goto err1; 1311 1312 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1313 if (!mw->vbase) { 1314 rc = -ENOMEM; 1315 goto err1; 1316 } 1317 1318 mw->buff_size = 0; 1319 mw->xlat_size = 0; 1320 mw->virt_addr = NULL; 1321 mw->dma_addr = 0; 1322 } 1323 1324 qp_bitmap = ntb_db_valid_mask(ndev); 1325 1326 qp_count = ilog2(qp_bitmap); 1327 if (nt->use_msi) { 1328 qp_count -= 1; 1329 nt->msi_db_mask = BIT_ULL(qp_count); 1330 ntb_db_clear_mask(ndev, nt->msi_db_mask); 1331 } 1332 1333 if (max_num_clients && max_num_clients < qp_count) 1334 qp_count = max_num_clients; 1335 else if (nt->mw_count < qp_count) 1336 qp_count = nt->mw_count; 1337 1338 qp_bitmap &= BIT_ULL(qp_count) - 1; 1339 1340 nt->qp_count = qp_count; 1341 nt->qp_bitmap = qp_bitmap; 1342 nt->qp_bitmap_free = qp_bitmap; 1343 1344 nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec), 1345 GFP_KERNEL, node); 1346 if (!nt->qp_vec) { 1347 rc = -ENOMEM; 1348 goto err1; 1349 } 1350 1351 if (nt_debugfs_dir) { 1352 nt->debugfs_node_dir = 1353 debugfs_create_dir(pci_name(ndev->pdev), 1354 nt_debugfs_dir); 1355 } 1356 1357 for (i = 0; i < qp_count; i++) { 1358 rc = ntb_transport_init_queue(nt, i); 1359 if (rc) 1360 goto err2; 1361 } 1362 1363 mutex_init(&nt->link_event_lock); 1364 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1365 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1366 1367 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); 1368 if (rc) 1369 goto err2; 1370 1371 INIT_LIST_HEAD(&nt->client_devs); 1372 rc = ntb_bus_init(nt); 1373 if (rc) 1374 goto err3; 1375 1376 nt->link_is_up = false; 1377 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1378 ntb_link_event(ndev); 1379 1380 return 0; 1381 1382 err3: 1383 ntb_clear_ctx(ndev); 1384 err2: 1385 kfree(nt->qp_vec); 1386 err1: 1387 while (i--) { 1388 mw = &nt->mw_vec[i]; 1389 iounmap(mw->vbase); 1390 } 1391 kfree(nt->mw_vec); 1392 err: 1393 kfree(nt); 1394 return rc; 1395 } 1396 1397 static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) 1398 { 1399 struct ntb_transport_ctx *nt = ndev->ctx; 1400 struct ntb_transport_qp *qp; 1401 u64 qp_bitmap_alloc; 1402 int i; 1403 1404 ntb_transport_link_cleanup(nt); 1405 cancel_work_sync(&nt->link_cleanup); 1406 cancel_delayed_work_sync(&nt->link_work); 1407 1408 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 1409 1410 /* verify that all the qp's are freed */ 1411 for (i = 0; i < nt->qp_count; i++) { 1412 qp = &nt->qp_vec[i]; 1413 if (qp_bitmap_alloc & BIT_ULL(i)) 1414 ntb_transport_free_queue(qp); 1415 debugfs_remove_recursive(qp->debugfs_dir); 1416 } 1417 1418 ntb_link_disable(ndev); 1419 ntb_clear_ctx(ndev); 1420 1421 ntb_bus_remove(nt); 1422 1423 for (i = nt->mw_count; i--; ) { 1424 ntb_free_mw(nt, i); 1425 iounmap(nt->mw_vec[i].vbase); 1426 } 1427 1428 kfree(nt->qp_vec); 1429 kfree(nt->mw_vec); 1430 kfree(nt); 1431 } 1432 1433 static void ntb_complete_rxc(struct ntb_transport_qp *qp) 1434 { 1435 struct ntb_queue_entry *entry; 1436 void *cb_data; 1437 unsigned int len; 1438 unsigned long irqflags; 1439 1440 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1441 1442 while (!list_empty(&qp->rx_post_q)) { 1443 entry = list_first_entry(&qp->rx_post_q, 1444 struct ntb_queue_entry, entry); 1445 if (!(entry->flags & DESC_DONE_FLAG)) 1446 break; 1447 1448 entry->rx_hdr->flags = 0; 1449 iowrite32(entry->rx_index, &qp->rx_info->entry); 1450 1451 cb_data = entry->cb_data; 1452 len = entry->len; 1453 1454 list_move_tail(&entry->entry, &qp->rx_free_q); 1455 1456 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1457 1458 if (qp->rx_handler && qp->client_ready) 1459 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1460 1461 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags); 1462 } 1463 1464 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags); 1465 } 1466 1467 static void ntb_rx_copy_callback(void *data, 1468 const struct dmaengine_result *res) 1469 { 1470 struct ntb_queue_entry *entry = data; 1471 1472 /* we need to check DMA results if we are using DMA */ 1473 if (res) { 1474 enum dmaengine_tx_result dma_err = res->result; 1475 1476 switch (dma_err) { 1477 case DMA_TRANS_READ_FAILED: 1478 case DMA_TRANS_WRITE_FAILED: 1479 entry->errors++; 1480 fallthrough; 1481 case DMA_TRANS_ABORTED: 1482 { 1483 struct ntb_transport_qp *qp = entry->qp; 1484 void *offset = qp->rx_buff + qp->rx_max_frame * 1485 qp->rx_index; 1486 1487 ntb_memcpy_rx(entry, offset); 1488 qp->rx_memcpy++; 1489 return; 1490 } 1491 1492 case DMA_TRANS_NOERROR: 1493 default: 1494 break; 1495 } 1496 } 1497 1498 entry->flags |= DESC_DONE_FLAG; 1499 1500 ntb_complete_rxc(entry->qp); 1501 } 1502 1503 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1504 { 1505 void *buf = entry->buf; 1506 size_t len = entry->len; 1507 1508 memcpy(buf, offset, len); 1509 1510 /* Ensure that the data is fully copied out before clearing the flag */ 1511 wmb(); 1512 1513 ntb_rx_copy_callback(entry, NULL); 1514 } 1515 1516 static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset) 1517 { 1518 struct dma_async_tx_descriptor *txd; 1519 struct ntb_transport_qp *qp = entry->qp; 1520 struct dma_chan *chan = qp->rx_dma_chan; 1521 struct dma_device *device; 1522 size_t pay_off, buff_off, len; 1523 struct dmaengine_unmap_data *unmap; 1524 dma_cookie_t cookie; 1525 void *buf = entry->buf; 1526 1527 len = entry->len; 1528 device = chan->device; 1529 pay_off = (size_t)offset & ~PAGE_MASK; 1530 buff_off = (size_t)buf & ~PAGE_MASK; 1531 1532 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1533 goto err; 1534 1535 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); 1536 if (!unmap) 1537 goto err; 1538 1539 unmap->len = len; 1540 unmap->addr[0] = dma_map_phys(device->dev, virt_to_phys(offset), 1541 len, DMA_TO_DEVICE, 0); 1542 if (dma_mapping_error(device->dev, unmap->addr[0])) 1543 goto err_get_unmap; 1544 1545 unmap->to_cnt = 1; 1546 1547 unmap->addr[1] = dma_map_phys(device->dev, virt_to_phys(buf), 1548 len, DMA_FROM_DEVICE, 0); 1549 if (dma_mapping_error(device->dev, unmap->addr[1])) 1550 goto err_get_unmap; 1551 1552 unmap->from_cnt = 1; 1553 1554 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1555 unmap->addr[0], len, 1556 DMA_PREP_INTERRUPT); 1557 if (!txd) 1558 goto err_get_unmap; 1559 1560 txd->callback_result = ntb_rx_copy_callback; 1561 txd->callback_param = entry; 1562 dma_set_unmap(txd, unmap); 1563 1564 cookie = dmaengine_submit(txd); 1565 if (dma_submit_error(cookie)) 1566 goto err_set_unmap; 1567 1568 dmaengine_unmap_put(unmap); 1569 1570 qp->last_cookie = cookie; 1571 1572 qp->rx_async++; 1573 1574 return 0; 1575 1576 err_set_unmap: 1577 dmaengine_unmap_put(unmap); 1578 err_get_unmap: 1579 dmaengine_unmap_put(unmap); 1580 err: 1581 return -ENXIO; 1582 } 1583 1584 static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset) 1585 { 1586 struct ntb_transport_qp *qp = entry->qp; 1587 struct dma_chan *chan = qp->rx_dma_chan; 1588 int res; 1589 1590 if (!chan) 1591 goto err; 1592 1593 if (entry->len < copy_bytes) 1594 goto err; 1595 1596 res = ntb_async_rx_submit(entry, offset); 1597 if (res < 0) 1598 goto err; 1599 1600 qp->rx_async++; 1601 return; 1602 1603 err: 1604 ntb_memcpy_rx(entry, offset); 1605 qp->rx_memcpy++; 1606 } 1607 1608 static int ntb_process_rxc(struct ntb_transport_qp *qp) 1609 { 1610 struct ntb_payload_header *hdr; 1611 struct ntb_queue_entry *entry; 1612 void *offset; 1613 1614 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1615 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1616 1617 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", 1618 qp->qp_num, hdr->ver, hdr->len, hdr->flags); 1619 1620 if (!(hdr->flags & DESC_DONE_FLAG)) { 1621 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); 1622 qp->rx_ring_empty++; 1623 return -EAGAIN; 1624 } 1625 1626 if (hdr->flags & LINK_DOWN_FLAG) { 1627 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); 1628 ntb_qp_link_down(qp); 1629 hdr->flags = 0; 1630 return -EAGAIN; 1631 } 1632 1633 if (hdr->ver != (u32)qp->rx_pkts) { 1634 dev_dbg(&qp->ndev->pdev->dev, 1635 "version mismatch, expected %llu - got %u\n", 1636 qp->rx_pkts, hdr->ver); 1637 qp->rx_err_ver++; 1638 return -EIO; 1639 } 1640 1641 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1642 if (!entry) { 1643 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); 1644 qp->rx_err_no_buf++; 1645 return -EAGAIN; 1646 } 1647 1648 entry->rx_hdr = hdr; 1649 entry->rx_index = qp->rx_index; 1650 1651 if (hdr->len > entry->len) { 1652 dev_dbg(&qp->ndev->pdev->dev, 1653 "receive buffer overflow! Wanted %d got %d\n", 1654 hdr->len, entry->len); 1655 qp->rx_err_oflow++; 1656 1657 entry->len = -EIO; 1658 entry->flags |= DESC_DONE_FLAG; 1659 1660 ntb_complete_rxc(qp); 1661 } else { 1662 dev_dbg(&qp->ndev->pdev->dev, 1663 "RX OK index %u ver %u size %d into buf size %d\n", 1664 qp->rx_index, hdr->ver, hdr->len, entry->len); 1665 1666 qp->rx_bytes += hdr->len; 1667 qp->rx_pkts++; 1668 1669 entry->len = hdr->len; 1670 1671 ntb_async_rx(entry, offset); 1672 } 1673 1674 qp->rx_index++; 1675 qp->rx_index %= qp->rx_max_entry; 1676 1677 return 0; 1678 } 1679 1680 static void ntb_transport_rxc_db(unsigned long data) 1681 { 1682 struct ntb_transport_qp *qp = (void *)data; 1683 int rc, i; 1684 1685 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", 1686 __func__, qp->qp_num); 1687 1688 /* Limit the number of packets processed in a single interrupt to 1689 * provide fairness to others 1690 */ 1691 for (i = 0; i < qp->rx_max_entry; i++) { 1692 rc = ntb_process_rxc(qp); 1693 if (rc) 1694 break; 1695 } 1696 1697 if (i && qp->rx_dma_chan) 1698 dma_async_issue_pending(qp->rx_dma_chan); 1699 1700 if (i == qp->rx_max_entry) { 1701 /* there is more work to do */ 1702 if (qp->active) 1703 tasklet_schedule(&qp->rxc_db_work); 1704 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { 1705 /* the doorbell bit is set: clear it */ 1706 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); 1707 /* ntb_db_read ensures ntb_db_clear write is committed */ 1708 ntb_db_read(qp->ndev); 1709 1710 /* an interrupt may have arrived between finishing 1711 * ntb_process_rxc and clearing the doorbell bit: 1712 * there might be some more work to do. 1713 */ 1714 if (qp->active) 1715 tasklet_schedule(&qp->rxc_db_work); 1716 } 1717 } 1718 1719 static void ntb_tx_copy_callback(void *data, 1720 const struct dmaengine_result *res) 1721 { 1722 struct ntb_queue_entry *entry = data; 1723 struct ntb_transport_qp *qp = entry->qp; 1724 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1725 1726 /* we need to check DMA results if we are using DMA */ 1727 if (res) { 1728 enum dmaengine_tx_result dma_err = res->result; 1729 1730 switch (dma_err) { 1731 case DMA_TRANS_READ_FAILED: 1732 case DMA_TRANS_WRITE_FAILED: 1733 entry->errors++; 1734 fallthrough; 1735 case DMA_TRANS_ABORTED: 1736 { 1737 void __iomem *offset = 1738 qp->tx_mw + qp->tx_max_frame * 1739 entry->tx_index; 1740 1741 /* resubmit via CPU */ 1742 ntb_memcpy_tx(entry, offset); 1743 qp->tx_memcpy++; 1744 return; 1745 } 1746 1747 case DMA_TRANS_NOERROR: 1748 default: 1749 break; 1750 } 1751 } 1752 1753 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1754 1755 /* 1756 * Make DONE flag visible before DB/MSI. WC + posted MWr may reorder 1757 * across iATU/bridge (platform-dependent). Order and flush here. 1758 */ 1759 dma_mb(); 1760 ioread32(&hdr->flags); 1761 1762 if (qp->use_msi) 1763 ntb_msi_peer_trigger(qp->ndev, PIDX, &qp->peer_msi_desc); 1764 else 1765 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); 1766 1767 /* The entry length can only be zero if the packet is intended to be a 1768 * "link down" or similar. Since no payload is being sent in these 1769 * cases, there is nothing to add to the completion queue. 1770 */ 1771 if (entry->len > 0) { 1772 qp->tx_bytes += entry->len; 1773 1774 if (qp->tx_handler) 1775 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1776 entry->len); 1777 } 1778 1779 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1780 } 1781 1782 static void ntb_memcpy_tx_on_stack(struct ntb_queue_entry *entry, void __iomem *offset) 1783 { 1784 #ifdef copy_to_nontemporal 1785 /* 1786 * Using non-temporal mov to improve performance on non-cached 1787 * writes. This only works if __iomem is strictly memory-like, 1788 * but that is the case on x86-64 1789 */ 1790 copy_to_nontemporal(offset, entry->buf, entry->len); 1791 #else 1792 memcpy_toio(offset, entry->buf, entry->len); 1793 #endif 1794 1795 /* Ensure that the data is fully copied out before setting the flags */ 1796 wmb(); 1797 1798 ntb_tx_copy_callback(entry, NULL); 1799 } 1800 1801 static int ntb_tx_memcpy_kthread(void *data) 1802 { 1803 struct ntb_transport_qp *qp = data; 1804 struct ntb_queue_entry *entry, *tmp; 1805 const int resched_nr = 64; 1806 LIST_HEAD(local_list); 1807 void __iomem *offset; 1808 int processed = 0; 1809 1810 while (!kthread_should_stop()) { 1811 spin_lock_irq(&qp->ntb_tx_offl_q_lock); 1812 wait_event_interruptible_lock_irq_timeout(qp->tx_offload_wq, 1813 kthread_should_stop() || 1814 !list_empty(&qp->tx_offl_q), 1815 qp->ntb_tx_offl_q_lock, 5*HZ); 1816 list_splice_tail_init(&qp->tx_offl_q, &local_list); 1817 spin_unlock_irq(&qp->ntb_tx_offl_q_lock); 1818 1819 list_for_each_entry_safe(entry, tmp, &local_list, entry) { 1820 list_del(&entry->entry); 1821 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; 1822 ntb_memcpy_tx_on_stack(entry, offset); 1823 if (++processed >= resched_nr) { 1824 cond_resched(); 1825 processed = 0; 1826 } 1827 } 1828 cond_resched(); 1829 } 1830 1831 return 0; 1832 } 1833 1834 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1835 { 1836 struct ntb_transport_qp *qp = entry->qp; 1837 1838 if (WARN_ON_ONCE(!qp)) 1839 return; 1840 1841 if (ntb_tx_offload_enabled(qp)) { 1842 ntb_list_add(&qp->ntb_tx_offl_q_lock, &entry->entry, 1843 &qp->tx_offl_q); 1844 wake_up(&qp->tx_offload_wq); 1845 } else 1846 ntb_memcpy_tx_on_stack(entry, offset); 1847 } 1848 1849 static int ntb_async_tx_submit(struct ntb_transport_qp *qp, 1850 struct ntb_queue_entry *entry) 1851 { 1852 struct dma_async_tx_descriptor *txd; 1853 struct dma_chan *chan = qp->tx_dma_chan; 1854 struct dma_device *device; 1855 size_t len = entry->len; 1856 void *buf = entry->buf; 1857 size_t dest_off, buff_off; 1858 struct dmaengine_unmap_data *unmap; 1859 dma_addr_t dest; 1860 dma_cookie_t cookie; 1861 1862 device = chan->device; 1863 dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index; 1864 buff_off = (size_t)buf & ~PAGE_MASK; 1865 dest_off = (size_t)dest & ~PAGE_MASK; 1866 1867 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1868 goto err; 1869 1870 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 1871 if (!unmap) 1872 goto err; 1873 1874 unmap->len = len; 1875 unmap->addr[0] = dma_map_phys(device->dev, virt_to_phys(buf), 1876 len, DMA_TO_DEVICE, 0); 1877 if (dma_mapping_error(device->dev, unmap->addr[0])) 1878 goto err_get_unmap; 1879 1880 unmap->to_cnt = 1; 1881 1882 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1883 DMA_PREP_INTERRUPT); 1884 if (!txd) 1885 goto err_get_unmap; 1886 1887 txd->callback_result = ntb_tx_copy_callback; 1888 txd->callback_param = entry; 1889 dma_set_unmap(txd, unmap); 1890 1891 cookie = dmaengine_submit(txd); 1892 if (dma_submit_error(cookie)) 1893 goto err_set_unmap; 1894 1895 dmaengine_unmap_put(unmap); 1896 1897 dma_async_issue_pending(chan); 1898 1899 return 0; 1900 err_set_unmap: 1901 dmaengine_unmap_put(unmap); 1902 err_get_unmap: 1903 dmaengine_unmap_put(unmap); 1904 err: 1905 return -ENXIO; 1906 } 1907 1908 static void ntb_async_tx(struct ntb_transport_qp *qp, 1909 struct ntb_queue_entry *entry) 1910 { 1911 struct ntb_payload_header __iomem *hdr; 1912 struct dma_chan *chan = qp->tx_dma_chan; 1913 void __iomem *offset; 1914 int res; 1915 1916 entry->tx_index = qp->tx_index; 1917 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index; 1918 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1919 entry->tx_hdr = hdr; 1920 1921 WARN_ON_ONCE(!ntb_transport_tx_free_entry(qp)); 1922 WRITE_ONCE(qp->tx_index, (qp->tx_index + 1) % qp->tx_max_entry); 1923 1924 iowrite32(entry->len, &hdr->len); 1925 iowrite32((u32)qp->tx_pkts, &hdr->ver); 1926 1927 if (!chan) 1928 goto err; 1929 1930 if (entry->len < copy_bytes) 1931 goto err; 1932 1933 res = ntb_async_tx_submit(qp, entry); 1934 if (res < 0) 1935 goto err; 1936 1937 qp->tx_async++; 1938 return; 1939 1940 err: 1941 ntb_memcpy_tx(entry, offset); 1942 qp->tx_memcpy++; 1943 } 1944 1945 static int ntb_process_tx(struct ntb_transport_qp *qp, 1946 struct ntb_queue_entry *entry) 1947 { 1948 if (!ntb_transport_tx_free_entry(qp)) { 1949 qp->tx_ring_full++; 1950 return -EAGAIN; 1951 } 1952 1953 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1954 if (qp->tx_handler) 1955 qp->tx_handler(qp, qp->cb_data, NULL, -EIO); 1956 1957 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1958 &qp->tx_free_q); 1959 return 0; 1960 } 1961 1962 ntb_async_tx(qp, entry); 1963 1964 qp->tx_pkts++; 1965 1966 return 0; 1967 } 1968 1969 static void ntb_send_link_down(struct ntb_transport_qp *qp) 1970 { 1971 struct pci_dev *pdev = qp->ndev->pdev; 1972 struct ntb_queue_entry *entry; 1973 int i, rc; 1974 1975 if (!qp->link_is_up) 1976 return; 1977 1978 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); 1979 1980 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1981 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1982 if (entry) 1983 break; 1984 msleep(100); 1985 } 1986 1987 if (!entry) 1988 return; 1989 1990 entry->cb_data = NULL; 1991 entry->buf = NULL; 1992 entry->len = 0; 1993 entry->flags = LINK_DOWN_FLAG; 1994 1995 rc = ntb_process_tx(qp, entry); 1996 if (rc) 1997 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1998 qp->qp_num); 1999 2000 ntb_qp_link_down_reset(qp); 2001 } 2002 2003 static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) 2004 { 2005 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 2006 } 2007 2008 /** 2009 * ntb_transport_create_queue - Create a new NTB transport layer queue 2010 * @data: pointer for callback data 2011 * @client_dev: &struct device pointer 2012 * @handlers: pointer to various ntb queue (callback) handlers 2013 * 2014 * Create a new NTB transport layer queue and provide the queue with a callback 2015 * routine for both transmit and receive. The receive callback routine will be 2016 * used to pass up data when the transport has received it on the queue. The 2017 * transmit callback routine will be called when the transport has completed the 2018 * transmission of the data on the queue and the data is ready to be freed. 2019 * 2020 * RETURNS: pointer to newly created ntb_queue, NULL on error. 2021 */ 2022 struct ntb_transport_qp * 2023 ntb_transport_create_queue(void *data, struct device *client_dev, 2024 const struct ntb_queue_handlers *handlers) 2025 { 2026 struct ntb_dev *ndev; 2027 struct pci_dev *pdev; 2028 struct ntb_transport_ctx *nt; 2029 struct ntb_queue_entry *entry; 2030 struct ntb_transport_qp *qp; 2031 u64 qp_bit; 2032 unsigned int free_queue; 2033 dma_cap_mask_t dma_mask; 2034 int node; 2035 int i; 2036 2037 ndev = dev_ntb(client_dev->parent); 2038 pdev = ndev->pdev; 2039 nt = ndev->ctx; 2040 2041 node = dev_to_node(&ndev->dev); 2042 2043 free_queue = ffs(nt->qp_bitmap_free); 2044 if (!free_queue) 2045 goto err; 2046 2047 /* decrement free_queue to make it zero based */ 2048 free_queue--; 2049 2050 qp = &nt->qp_vec[free_queue]; 2051 qp_bit = BIT_ULL(qp->qp_num); 2052 2053 nt->qp_bitmap_free &= ~qp_bit; 2054 2055 qp->cb_data = data; 2056 qp->rx_handler = handlers->rx_handler; 2057 qp->tx_handler = handlers->tx_handler; 2058 qp->event_handler = handlers->event_handler; 2059 2060 init_waitqueue_head(&qp->tx_offload_wq); 2061 if (tx_memcpy_offload) { 2062 qp->tx_offload_thread = kthread_run(ntb_tx_memcpy_kthread, qp, 2063 "ntb-txcpy/%s/%u", 2064 pci_name(ndev->pdev), qp->qp_num); 2065 if (IS_ERR(qp->tx_offload_thread)) { 2066 dev_warn(&nt->ndev->dev, 2067 "tx memcpy offload thread creation failed: %ld; falling back to inline copy\n", 2068 PTR_ERR(qp->tx_offload_thread)); 2069 qp->tx_offload_thread = NULL; 2070 } 2071 } else 2072 qp->tx_offload_thread = NULL; 2073 2074 dma_cap_zero(dma_mask); 2075 dma_cap_set(DMA_MEMCPY, dma_mask); 2076 2077 if (use_dma) { 2078 qp->tx_dma_chan = 2079 dma_request_channel(dma_mask, ntb_dma_filter_fn, 2080 (void *)(unsigned long)node); 2081 if (!qp->tx_dma_chan) 2082 dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n"); 2083 2084 qp->rx_dma_chan = 2085 dma_request_channel(dma_mask, ntb_dma_filter_fn, 2086 (void *)(unsigned long)node); 2087 if (!qp->rx_dma_chan) 2088 dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n"); 2089 } else { 2090 qp->tx_dma_chan = NULL; 2091 qp->rx_dma_chan = NULL; 2092 } 2093 2094 qp->tx_mw_dma_addr = 0; 2095 if (qp->tx_dma_chan) { 2096 qp->tx_mw_dma_addr = 2097 dma_map_resource(qp->tx_dma_chan->device->dev, 2098 qp->tx_mw_phys, qp->tx_mw_size, 2099 DMA_FROM_DEVICE, 0); 2100 if (dma_mapping_error(qp->tx_dma_chan->device->dev, 2101 qp->tx_mw_dma_addr)) { 2102 qp->tx_mw_dma_addr = 0; 2103 goto err1; 2104 } 2105 } 2106 2107 dev_dbg(&pdev->dev, "Using %s memcpy for TX\n", 2108 qp->tx_dma_chan ? "DMA" : "CPU"); 2109 2110 dev_dbg(&pdev->dev, "Using %s memcpy for RX\n", 2111 qp->rx_dma_chan ? "DMA" : "CPU"); 2112 2113 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 2114 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 2115 if (!entry) 2116 goto err1; 2117 2118 entry->qp = qp; 2119 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, 2120 &qp->rx_free_q); 2121 } 2122 qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; 2123 2124 for (i = 0; i < qp->tx_max_entry; i++) { 2125 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); 2126 if (!entry) 2127 goto err2; 2128 2129 entry->qp = qp; 2130 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2131 &qp->tx_free_q); 2132 } 2133 2134 ntb_db_clear(qp->ndev, qp_bit); 2135 ntb_db_clear_mask(qp->ndev, qp_bit); 2136 2137 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 2138 2139 return qp; 2140 2141 err2: 2142 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 2143 kfree(entry); 2144 err1: 2145 qp->rx_alloc_entry = 0; 2146 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 2147 kfree(entry); 2148 if (qp->tx_mw_dma_addr) 2149 dma_unmap_resource(qp->tx_dma_chan->device->dev, 2150 qp->tx_mw_dma_addr, qp->tx_mw_size, 2151 DMA_FROM_DEVICE, 0); 2152 if (qp->tx_dma_chan) 2153 dma_release_channel(qp->tx_dma_chan); 2154 if (qp->rx_dma_chan) 2155 dma_release_channel(qp->rx_dma_chan); 2156 nt->qp_bitmap_free |= qp_bit; 2157 err: 2158 return NULL; 2159 } 2160 EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 2161 2162 /** 2163 * ntb_transport_free_queue - Frees NTB transport queue 2164 * @qp: NTB queue to be freed 2165 * 2166 * Frees NTB transport queue 2167 */ 2168 void ntb_transport_free_queue(struct ntb_transport_qp *qp) 2169 { 2170 struct pci_dev *pdev; 2171 struct ntb_queue_entry *entry; 2172 u64 qp_bit; 2173 2174 if (!qp) 2175 return; 2176 2177 pdev = qp->ndev->pdev; 2178 2179 qp->active = false; 2180 2181 if (qp->tx_offload_thread) { 2182 kthread_stop(qp->tx_offload_thread); 2183 qp->tx_offload_thread = NULL; 2184 } 2185 2186 if (qp->tx_dma_chan) { 2187 struct dma_chan *chan = qp->tx_dma_chan; 2188 /* Putting the dma_chan to NULL will force any new traffic to be 2189 * processed by the CPU instead of the DAM engine 2190 */ 2191 qp->tx_dma_chan = NULL; 2192 2193 /* Try to be nice and wait for any queued DMA engine 2194 * transactions to process before smashing it with a rock 2195 */ 2196 dma_sync_wait(chan, qp->last_cookie); 2197 dmaengine_terminate_all(chan); 2198 2199 dma_unmap_resource(chan->device->dev, 2200 qp->tx_mw_dma_addr, qp->tx_mw_size, 2201 DMA_FROM_DEVICE, 0); 2202 2203 dma_release_channel(chan); 2204 } 2205 2206 if (qp->rx_dma_chan) { 2207 struct dma_chan *chan = qp->rx_dma_chan; 2208 /* Putting the dma_chan to NULL will force any new traffic to be 2209 * processed by the CPU instead of the DAM engine 2210 */ 2211 qp->rx_dma_chan = NULL; 2212 2213 /* Try to be nice and wait for any queued DMA engine 2214 * transactions to process before smashing it with a rock 2215 */ 2216 dma_sync_wait(chan, qp->last_cookie); 2217 dmaengine_terminate_all(chan); 2218 dma_release_channel(chan); 2219 } 2220 2221 qp_bit = BIT_ULL(qp->qp_num); 2222 2223 ntb_db_set_mask(qp->ndev, qp_bit); 2224 tasklet_kill(&qp->rxc_db_work); 2225 2226 cancel_delayed_work_sync(&qp->link_work); 2227 2228 qp->cb_data = NULL; 2229 qp->rx_handler = NULL; 2230 qp->tx_handler = NULL; 2231 qp->event_handler = NULL; 2232 2233 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 2234 kfree(entry); 2235 2236 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) { 2237 dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n"); 2238 kfree(entry); 2239 } 2240 2241 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) { 2242 dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n"); 2243 kfree(entry); 2244 } 2245 2246 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 2247 kfree(entry); 2248 2249 while ((entry = ntb_list_rm(&qp->ntb_tx_offl_q_lock, &qp->tx_offl_q))) 2250 kfree(entry); 2251 2252 qp->transport->qp_bitmap_free |= qp_bit; 2253 2254 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 2255 } 2256 EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 2257 2258 /** 2259 * ntb_transport_rx_remove - Dequeues enqueued rx packet 2260 * @qp: NTB queue to be freed 2261 * @len: pointer to variable to write enqueued buffers length 2262 * 2263 * Dequeues unused buffers from receive queue. Should only be used during 2264 * shutdown of qp. 2265 * 2266 * RETURNS: NULL error value on error, or void* for success. 2267 */ 2268 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 2269 { 2270 struct ntb_queue_entry *entry; 2271 void *buf; 2272 2273 if (!qp || qp->client_ready) 2274 return NULL; 2275 2276 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q); 2277 if (!entry) 2278 return NULL; 2279 2280 buf = entry->cb_data; 2281 *len = entry->len; 2282 2283 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q); 2284 2285 return buf; 2286 } 2287 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 2288 2289 /** 2290 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 2291 * @qp: NTB transport layer queue the entry is to be enqueued on 2292 * @cb: per buffer pointer for callback function to use 2293 * @data: pointer to data buffer that incoming packets will be copied into 2294 * @len: length of the data buffer 2295 * 2296 * Enqueue a new receive buffer onto the transport queue into which a NTB 2297 * payload can be received into. 2298 * 2299 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2300 */ 2301 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2302 unsigned int len) 2303 { 2304 struct ntb_queue_entry *entry; 2305 2306 if (!qp) 2307 return -EINVAL; 2308 2309 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q); 2310 if (!entry) 2311 return -ENOMEM; 2312 2313 entry->cb_data = cb; 2314 entry->buf = data; 2315 entry->len = len; 2316 entry->flags = 0; 2317 entry->errors = 0; 2318 entry->rx_index = 0; 2319 2320 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q); 2321 2322 if (qp->active) 2323 tasklet_schedule(&qp->rxc_db_work); 2324 2325 return 0; 2326 } 2327 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 2328 2329 /** 2330 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 2331 * @qp: NTB transport layer queue the entry is to be enqueued on 2332 * @cb: per buffer pointer for callback function to use 2333 * @data: pointer to data buffer that will be sent 2334 * @len: length of the data buffer 2335 * 2336 * Enqueue a new transmit buffer onto the transport queue from which a NTB 2337 * payload will be transmitted. This assumes that a lock is being held to 2338 * serialize access to the qp. 2339 * 2340 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 2341 */ 2342 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 2343 unsigned int len) 2344 { 2345 struct ntb_queue_entry *entry; 2346 int rc; 2347 2348 if (!qp || !len) 2349 return -EINVAL; 2350 2351 /* If the qp link is down already, just ignore. */ 2352 if (!qp->link_is_up) 2353 return 0; 2354 2355 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 2356 if (!entry) { 2357 qp->tx_err_no_buf++; 2358 return -EBUSY; 2359 } 2360 2361 entry->cb_data = cb; 2362 entry->buf = data; 2363 entry->len = len; 2364 entry->flags = 0; 2365 entry->errors = 0; 2366 entry->tx_index = 0; 2367 2368 rc = ntb_process_tx(qp, entry); 2369 if (rc) 2370 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 2371 &qp->tx_free_q); 2372 2373 return rc; 2374 } 2375 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 2376 2377 /** 2378 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 2379 * @qp: NTB transport layer queue to be enabled 2380 * 2381 * Notify NTB transport layer of client readiness to use queue 2382 */ 2383 void ntb_transport_link_up(struct ntb_transport_qp *qp) 2384 { 2385 if (!qp) 2386 return; 2387 2388 qp->client_ready = true; 2389 2390 if (qp->transport->link_is_up) 2391 schedule_delayed_work(&qp->link_work, 0); 2392 } 2393 EXPORT_SYMBOL_GPL(ntb_transport_link_up); 2394 2395 /** 2396 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 2397 * @qp: NTB transport layer queue to be disabled 2398 * 2399 * Notify NTB transport layer of client's desire to no longer receive data on 2400 * transport queue specified. It is the client's responsibility to ensure all 2401 * entries on queue are purged or otherwise handled appropriately. 2402 */ 2403 void ntb_transport_link_down(struct ntb_transport_qp *qp) 2404 { 2405 int val; 2406 2407 if (!qp) 2408 return; 2409 2410 qp->client_ready = false; 2411 2412 val = ntb_spad_read(qp->ndev, QP_LINKS); 2413 2414 ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num)); 2415 2416 if (qp->link_is_up) 2417 ntb_send_link_down(qp); 2418 else 2419 cancel_delayed_work_sync(&qp->link_work); 2420 } 2421 EXPORT_SYMBOL_GPL(ntb_transport_link_down); 2422 2423 /** 2424 * ntb_transport_link_query - Query transport link state 2425 * @qp: NTB transport layer queue to be queried 2426 * 2427 * Query connectivity to the remote system of the NTB transport queue 2428 * 2429 * RETURNS: true for link up or false for link down 2430 */ 2431 bool ntb_transport_link_query(struct ntb_transport_qp *qp) 2432 { 2433 if (!qp) 2434 return false; 2435 2436 return qp->link_is_up; 2437 } 2438 EXPORT_SYMBOL_GPL(ntb_transport_link_query); 2439 2440 /** 2441 * ntb_transport_qp_num - Query the qp number 2442 * @qp: NTB transport layer queue to be queried 2443 * 2444 * Query qp number of the NTB transport queue 2445 * 2446 * RETURNS: a zero based number specifying the qp number 2447 */ 2448 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 2449 { 2450 if (!qp) 2451 return 0; 2452 2453 return qp->qp_num; 2454 } 2455 EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 2456 2457 /** 2458 * ntb_transport_max_size - Query the max payload size of a qp 2459 * @qp: NTB transport layer queue to be queried 2460 * 2461 * Query the maximum payload size permissible on the given qp 2462 * 2463 * RETURNS: the max payload size of a qp 2464 */ 2465 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 2466 { 2467 unsigned int max_size; 2468 unsigned int copy_align; 2469 struct dma_chan *rx_chan, *tx_chan; 2470 2471 if (!qp) 2472 return 0; 2473 2474 rx_chan = qp->rx_dma_chan; 2475 tx_chan = qp->tx_dma_chan; 2476 2477 copy_align = max(rx_chan ? rx_chan->device->copy_align : 0, 2478 tx_chan ? tx_chan->device->copy_align : 0); 2479 2480 /* If DMA engine usage is possible, try to find the max size for that */ 2481 max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header); 2482 max_size = round_down(max_size, 1 << copy_align); 2483 2484 return max_size; 2485 } 2486 EXPORT_SYMBOL_GPL(ntb_transport_max_size); 2487 2488 unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp) 2489 { 2490 unsigned int head = qp->tx_index; 2491 unsigned int tail = qp->remote_rx_info->entry; 2492 2493 return tail >= head ? tail - head : qp->tx_max_entry + tail - head; 2494 } 2495 EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry); 2496 2497 static void ntb_transport_doorbell_callback(void *data, int vector) 2498 { 2499 struct ntb_transport_ctx *nt = data; 2500 struct ntb_transport_qp *qp; 2501 u64 db_bits; 2502 unsigned int qp_num; 2503 2504 if (ntb_db_read(nt->ndev) & nt->msi_db_mask) { 2505 ntb_transport_msi_peer_desc_changed(nt); 2506 ntb_db_clear(nt->ndev, nt->msi_db_mask); 2507 } 2508 2509 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & 2510 ntb_db_vector_mask(nt->ndev, vector)); 2511 2512 while (db_bits) { 2513 qp_num = __ffs(db_bits); 2514 qp = &nt->qp_vec[qp_num]; 2515 2516 if (qp->active) 2517 tasklet_schedule(&qp->rxc_db_work); 2518 2519 db_bits &= ~BIT_ULL(qp_num); 2520 } 2521 } 2522 2523 static const struct ntb_ctx_ops ntb_transport_ops = { 2524 .link_event = ntb_transport_event_callback, 2525 .db_event = ntb_transport_doorbell_callback, 2526 }; 2527 2528 static struct ntb_client ntb_transport_client = { 2529 .ops = { 2530 .probe = ntb_transport_probe, 2531 .remove = ntb_transport_free, 2532 }, 2533 }; 2534 2535 static int __init ntb_transport_init(void) 2536 { 2537 int rc; 2538 2539 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); 2540 2541 if (debugfs_initialized()) 2542 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2543 2544 rc = bus_register(&ntb_transport_bus); 2545 if (rc) 2546 goto err_bus; 2547 2548 rc = ntb_register_client(&ntb_transport_client); 2549 if (rc) 2550 goto err_client; 2551 2552 return 0; 2553 2554 err_client: 2555 bus_unregister(&ntb_transport_bus); 2556 err_bus: 2557 debugfs_remove_recursive(nt_debugfs_dir); 2558 return rc; 2559 } 2560 module_init(ntb_transport_init); 2561 2562 static void __exit ntb_transport_exit(void) 2563 { 2564 ntb_unregister_client(&ntb_transport_client); 2565 bus_unregister(&ntb_transport_bus); 2566 debugfs_remove_recursive(nt_debugfs_dir); 2567 } 2568 module_exit(ntb_transport_exit); 2569