1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * BSD LICENSE 15 * 16 * Copyright(c) 2012 Intel Corporation. All rights reserved. 17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 23 * * Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * * Redistributions in binary form must reproduce the above copy 26 * notice, this list of conditions and the following disclaimer in 27 * the documentation and/or other materials provided with the 28 * distribution. 29 * * Neither the name of Intel Corporation nor the names of its 30 * contributors may be used to endorse or promote products derived 31 * from this software without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 * 45 * PCIe NTB Transport Linux driver 46 * 47 * Contact Information: 48 * Jon Mason <jon.mason@intel.com> 49 */ 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/dmaengine.h> 53 #include <linux/dma-mapping.h> 54 #include <linux/errno.h> 55 #include <linux/export.h> 56 #include <linux/interrupt.h> 57 #include <linux/module.h> 58 #include <linux/pci.h> 59 #include <linux/slab.h> 60 #include <linux/types.h> 61 #include <linux/uaccess.h> 62 #include "linux/ntb.h" 63 #include "linux/ntb_transport.h" 64 65 #define NTB_TRANSPORT_VERSION 4 66 #define NTB_TRANSPORT_VER "4" 67 #define NTB_TRANSPORT_NAME "ntb_transport" 68 #define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB" 69 70 MODULE_DESCRIPTION(NTB_TRANSPORT_DESC); 71 MODULE_VERSION(NTB_TRANSPORT_VER); 72 MODULE_LICENSE("Dual BSD/GPL"); 73 MODULE_AUTHOR("Intel Corporation"); 74 75 static unsigned long max_mw_size; 76 module_param(max_mw_size, ulong, 0644); 77 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 78 79 static unsigned int transport_mtu = 0x10000; 80 module_param(transport_mtu, uint, 0644); 81 MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); 82 83 static unsigned char max_num_clients; 84 module_param(max_num_clients, byte, 0644); 85 MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); 86 87 static unsigned int copy_bytes = 1024; 88 module_param(copy_bytes, uint, 0644); 89 MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 90 91 static bool use_dma; 92 module_param(use_dma, bool, 0644); 93 MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy"); 94 95 static struct dentry *nt_debugfs_dir; 96 97 struct ntb_queue_entry { 98 /* ntb_queue list reference */ 99 struct list_head entry; 100 /* pointers to data to be transferred */ 101 void *cb_data; 102 void *buf; 103 unsigned int len; 104 unsigned int flags; 105 106 struct ntb_transport_qp *qp; 107 union { 108 struct ntb_payload_header __iomem *tx_hdr; 109 struct ntb_payload_header *rx_hdr; 110 }; 111 unsigned int index; 112 }; 113 114 struct ntb_rx_info { 115 unsigned int entry; 116 }; 117 118 struct ntb_transport_qp { 119 struct ntb_transport_ctx *transport; 120 struct ntb_dev *ndev; 121 void *cb_data; 122 struct dma_chan *dma_chan; 123 124 bool client_ready; 125 bool link_is_up; 126 127 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 128 u64 qp_bit; 129 130 struct ntb_rx_info __iomem *rx_info; 131 struct ntb_rx_info *remote_rx_info; 132 133 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 134 void *data, int len); 135 struct list_head tx_free_q; 136 spinlock_t ntb_tx_free_q_lock; 137 void __iomem *tx_mw; 138 dma_addr_t tx_mw_phys; 139 unsigned int tx_index; 140 unsigned int tx_max_entry; 141 unsigned int tx_max_frame; 142 143 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 144 void *data, int len); 145 struct list_head rx_pend_q; 146 struct list_head rx_free_q; 147 spinlock_t ntb_rx_pend_q_lock; 148 spinlock_t ntb_rx_free_q_lock; 149 void *rx_buff; 150 unsigned int rx_index; 151 unsigned int rx_max_entry; 152 unsigned int rx_max_frame; 153 dma_cookie_t last_cookie; 154 struct tasklet_struct rxc_db_work; 155 156 void (*event_handler)(void *data, int status); 157 struct delayed_work link_work; 158 struct work_struct link_cleanup; 159 160 struct dentry *debugfs_dir; 161 struct dentry *debugfs_stats; 162 163 /* Stats */ 164 u64 rx_bytes; 165 u64 rx_pkts; 166 u64 rx_ring_empty; 167 u64 rx_err_no_buf; 168 u64 rx_err_oflow; 169 u64 rx_err_ver; 170 u64 rx_memcpy; 171 u64 rx_async; 172 u64 tx_bytes; 173 u64 tx_pkts; 174 u64 tx_ring_full; 175 u64 tx_err_no_buf; 176 u64 tx_memcpy; 177 u64 tx_async; 178 }; 179 180 struct ntb_transport_mw { 181 phys_addr_t phys_addr; 182 resource_size_t phys_size; 183 resource_size_t xlat_align; 184 resource_size_t xlat_align_size; 185 void __iomem *vbase; 186 size_t xlat_size; 187 size_t buff_size; 188 void *virt_addr; 189 dma_addr_t dma_addr; 190 }; 191 192 struct ntb_transport_client_dev { 193 struct list_head entry; 194 struct ntb_transport_ctx *nt; 195 struct device dev; 196 }; 197 198 struct ntb_transport_ctx { 199 struct list_head entry; 200 struct list_head client_devs; 201 202 struct ntb_dev *ndev; 203 204 struct ntb_transport_mw *mw_vec; 205 struct ntb_transport_qp *qp_vec; 206 unsigned int mw_count; 207 unsigned int qp_count; 208 u64 qp_bitmap; 209 u64 qp_bitmap_free; 210 211 bool link_is_up; 212 struct delayed_work link_work; 213 struct work_struct link_cleanup; 214 }; 215 216 enum { 217 DESC_DONE_FLAG = BIT(0), 218 LINK_DOWN_FLAG = BIT(1), 219 }; 220 221 struct ntb_payload_header { 222 unsigned int ver; 223 unsigned int len; 224 unsigned int flags; 225 }; 226 227 enum { 228 VERSION = 0, 229 QP_LINKS, 230 NUM_QPS, 231 NUM_MWS, 232 MW0_SZ_HIGH, 233 MW0_SZ_LOW, 234 MW1_SZ_HIGH, 235 MW1_SZ_LOW, 236 MAX_SPAD, 237 }; 238 239 #define dev_client_dev(__dev) \ 240 container_of((__dev), struct ntb_transport_client_dev, dev) 241 242 #define drv_client(__drv) \ 243 container_of((__drv), struct ntb_transport_client, driver) 244 245 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 246 #define NTB_QP_DEF_NUM_ENTRIES 100 247 #define NTB_LINK_DOWN_TIMEOUT 10 248 249 static void ntb_transport_rxc_db(unsigned long data); 250 static const struct ntb_ctx_ops ntb_transport_ops; 251 static struct ntb_client ntb_transport_client; 252 253 static int ntb_transport_bus_match(struct device *dev, 254 struct device_driver *drv) 255 { 256 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 257 } 258 259 static int ntb_transport_bus_probe(struct device *dev) 260 { 261 const struct ntb_transport_client *client; 262 int rc = -EINVAL; 263 264 get_device(dev); 265 266 client = drv_client(dev->driver); 267 rc = client->probe(dev); 268 if (rc) 269 put_device(dev); 270 271 return rc; 272 } 273 274 static int ntb_transport_bus_remove(struct device *dev) 275 { 276 const struct ntb_transport_client *client; 277 278 client = drv_client(dev->driver); 279 client->remove(dev); 280 281 put_device(dev); 282 283 return 0; 284 } 285 286 static struct bus_type ntb_transport_bus = { 287 .name = "ntb_transport", 288 .match = ntb_transport_bus_match, 289 .probe = ntb_transport_bus_probe, 290 .remove = ntb_transport_bus_remove, 291 }; 292 293 static LIST_HEAD(ntb_transport_list); 294 295 static int ntb_bus_init(struct ntb_transport_ctx *nt) 296 { 297 list_add(&nt->entry, &ntb_transport_list); 298 return 0; 299 } 300 301 static void ntb_bus_remove(struct ntb_transport_ctx *nt) 302 { 303 struct ntb_transport_client_dev *client_dev, *cd; 304 305 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { 306 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", 307 dev_name(&client_dev->dev)); 308 list_del(&client_dev->entry); 309 device_unregister(&client_dev->dev); 310 } 311 312 list_del(&nt->entry); 313 } 314 315 static void ntb_transport_client_release(struct device *dev) 316 { 317 struct ntb_transport_client_dev *client_dev; 318 319 client_dev = dev_client_dev(dev); 320 kfree(client_dev); 321 } 322 323 /** 324 * ntb_transport_unregister_client_dev - Unregister NTB client device 325 * @device_name: Name of NTB client device 326 * 327 * Unregister an NTB client device with the NTB transport layer 328 */ 329 void ntb_transport_unregister_client_dev(char *device_name) 330 { 331 struct ntb_transport_client_dev *client, *cd; 332 struct ntb_transport_ctx *nt; 333 334 list_for_each_entry(nt, &ntb_transport_list, entry) 335 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 336 if (!strncmp(dev_name(&client->dev), device_name, 337 strlen(device_name))) { 338 list_del(&client->entry); 339 device_unregister(&client->dev); 340 } 341 } 342 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev); 343 344 /** 345 * ntb_transport_register_client_dev - Register NTB client device 346 * @device_name: Name of NTB client device 347 * 348 * Register an NTB client device with the NTB transport layer 349 */ 350 int ntb_transport_register_client_dev(char *device_name) 351 { 352 struct ntb_transport_client_dev *client_dev; 353 struct ntb_transport_ctx *nt; 354 int node; 355 int rc, i = 0; 356 357 if (list_empty(&ntb_transport_list)) 358 return -ENODEV; 359 360 list_for_each_entry(nt, &ntb_transport_list, entry) { 361 struct device *dev; 362 363 node = dev_to_node(&nt->ndev->dev); 364 365 client_dev = kzalloc_node(sizeof(*client_dev), 366 GFP_KERNEL, node); 367 if (!client_dev) { 368 rc = -ENOMEM; 369 goto err; 370 } 371 372 dev = &client_dev->dev; 373 374 /* setup and register client devices */ 375 dev_set_name(dev, "%s%d", device_name, i); 376 dev->bus = &ntb_transport_bus; 377 dev->release = ntb_transport_client_release; 378 dev->parent = &nt->ndev->dev; 379 380 rc = device_register(dev); 381 if (rc) { 382 kfree(client_dev); 383 goto err; 384 } 385 386 list_add_tail(&client_dev->entry, &nt->client_devs); 387 i++; 388 } 389 390 return 0; 391 392 err: 393 ntb_transport_unregister_client_dev(device_name); 394 395 return rc; 396 } 397 EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev); 398 399 /** 400 * ntb_transport_register_client - Register NTB client driver 401 * @drv: NTB client driver to be registered 402 * 403 * Register an NTB client driver with the NTB transport layer 404 * 405 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 406 */ 407 int ntb_transport_register_client(struct ntb_transport_client *drv) 408 { 409 drv->driver.bus = &ntb_transport_bus; 410 411 if (list_empty(&ntb_transport_list)) 412 return -ENODEV; 413 414 return driver_register(&drv->driver); 415 } 416 EXPORT_SYMBOL_GPL(ntb_transport_register_client); 417 418 /** 419 * ntb_transport_unregister_client - Unregister NTB client driver 420 * @drv: NTB client driver to be unregistered 421 * 422 * Unregister an NTB client driver with the NTB transport layer 423 * 424 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 425 */ 426 void ntb_transport_unregister_client(struct ntb_transport_client *drv) 427 { 428 driver_unregister(&drv->driver); 429 } 430 EXPORT_SYMBOL_GPL(ntb_transport_unregister_client); 431 432 static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, 433 loff_t *offp) 434 { 435 struct ntb_transport_qp *qp; 436 char *buf; 437 ssize_t ret, out_offset, out_count; 438 439 out_count = 1000; 440 441 buf = kmalloc(out_count, GFP_KERNEL); 442 if (!buf) 443 return -ENOMEM; 444 445 qp = filp->private_data; 446 out_offset = 0; 447 out_offset += snprintf(buf + out_offset, out_count - out_offset, 448 "NTB QP stats\n"); 449 out_offset += snprintf(buf + out_offset, out_count - out_offset, 450 "rx_bytes - \t%llu\n", qp->rx_bytes); 451 out_offset += snprintf(buf + out_offset, out_count - out_offset, 452 "rx_pkts - \t%llu\n", qp->rx_pkts); 453 out_offset += snprintf(buf + out_offset, out_count - out_offset, 454 "rx_memcpy - \t%llu\n", qp->rx_memcpy); 455 out_offset += snprintf(buf + out_offset, out_count - out_offset, 456 "rx_async - \t%llu\n", qp->rx_async); 457 out_offset += snprintf(buf + out_offset, out_count - out_offset, 458 "rx_ring_empty - %llu\n", qp->rx_ring_empty); 459 out_offset += snprintf(buf + out_offset, out_count - out_offset, 460 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); 461 out_offset += snprintf(buf + out_offset, out_count - out_offset, 462 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); 463 out_offset += snprintf(buf + out_offset, out_count - out_offset, 464 "rx_err_ver - \t%llu\n", qp->rx_err_ver); 465 out_offset += snprintf(buf + out_offset, out_count - out_offset, 466 "rx_buff - \t%p\n", qp->rx_buff); 467 out_offset += snprintf(buf + out_offset, out_count - out_offset, 468 "rx_index - \t%u\n", qp->rx_index); 469 out_offset += snprintf(buf + out_offset, out_count - out_offset, 470 "rx_max_entry - \t%u\n", qp->rx_max_entry); 471 472 out_offset += snprintf(buf + out_offset, out_count - out_offset, 473 "tx_bytes - \t%llu\n", qp->tx_bytes); 474 out_offset += snprintf(buf + out_offset, out_count - out_offset, 475 "tx_pkts - \t%llu\n", qp->tx_pkts); 476 out_offset += snprintf(buf + out_offset, out_count - out_offset, 477 "tx_memcpy - \t%llu\n", qp->tx_memcpy); 478 out_offset += snprintf(buf + out_offset, out_count - out_offset, 479 "tx_async - \t%llu\n", qp->tx_async); 480 out_offset += snprintf(buf + out_offset, out_count - out_offset, 481 "tx_ring_full - \t%llu\n", qp->tx_ring_full); 482 out_offset += snprintf(buf + out_offset, out_count - out_offset, 483 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); 484 out_offset += snprintf(buf + out_offset, out_count - out_offset, 485 "tx_mw - \t%p\n", qp->tx_mw); 486 out_offset += snprintf(buf + out_offset, out_count - out_offset, 487 "tx_index - \t%u\n", qp->tx_index); 488 out_offset += snprintf(buf + out_offset, out_count - out_offset, 489 "tx_max_entry - \t%u\n", qp->tx_max_entry); 490 491 out_offset += snprintf(buf + out_offset, out_count - out_offset, 492 "\nQP Link %s\n", 493 qp->link_is_up ? "Up" : "Down"); 494 if (out_offset > out_count) 495 out_offset = out_count; 496 497 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 498 kfree(buf); 499 return ret; 500 } 501 502 static const struct file_operations ntb_qp_debugfs_stats = { 503 .owner = THIS_MODULE, 504 .open = simple_open, 505 .read = debugfs_read, 506 }; 507 508 static void ntb_list_add(spinlock_t *lock, struct list_head *entry, 509 struct list_head *list) 510 { 511 unsigned long flags; 512 513 spin_lock_irqsave(lock, flags); 514 list_add_tail(entry, list); 515 spin_unlock_irqrestore(lock, flags); 516 } 517 518 static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, 519 struct list_head *list) 520 { 521 struct ntb_queue_entry *entry; 522 unsigned long flags; 523 524 spin_lock_irqsave(lock, flags); 525 if (list_empty(list)) { 526 entry = NULL; 527 goto out; 528 } 529 entry = list_first_entry(list, struct ntb_queue_entry, entry); 530 list_del(&entry->entry); 531 out: 532 spin_unlock_irqrestore(lock, flags); 533 534 return entry; 535 } 536 537 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 538 unsigned int qp_num) 539 { 540 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 541 struct ntb_transport_mw *mw; 542 unsigned int rx_size, num_qps_mw; 543 unsigned int mw_num, mw_count, qp_count; 544 unsigned int i; 545 546 mw_count = nt->mw_count; 547 qp_count = nt->qp_count; 548 549 mw_num = QP_TO_MW(nt, qp_num); 550 mw = &nt->mw_vec[mw_num]; 551 552 if (!mw->virt_addr) 553 return -ENOMEM; 554 555 if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) 556 num_qps_mw = qp_count / mw_count + 1; 557 else 558 num_qps_mw = qp_count / mw_count; 559 560 rx_size = (unsigned int)mw->xlat_size / num_qps_mw; 561 qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count; 562 rx_size -= sizeof(struct ntb_rx_info); 563 564 qp->remote_rx_info = qp->rx_buff + rx_size; 565 566 /* Due to housekeeping, there must be atleast 2 buffs */ 567 qp->rx_max_frame = min(transport_mtu, rx_size / 2); 568 qp->rx_max_entry = rx_size / qp->rx_max_frame; 569 qp->rx_index = 0; 570 571 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 572 573 /* setup the hdr offsets with 0's */ 574 for (i = 0; i < qp->rx_max_entry; i++) { 575 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) - 576 sizeof(struct ntb_payload_header)); 577 memset(offset, 0, sizeof(struct ntb_payload_header)); 578 } 579 580 qp->rx_pkts = 0; 581 qp->tx_pkts = 0; 582 qp->tx_index = 0; 583 584 return 0; 585 } 586 587 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 588 { 589 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 590 struct pci_dev *pdev = nt->ndev->pdev; 591 592 if (!mw->virt_addr) 593 return; 594 595 ntb_mw_clear_trans(nt->ndev, num_mw); 596 dma_free_coherent(&pdev->dev, mw->buff_size, 597 mw->virt_addr, mw->dma_addr); 598 mw->xlat_size = 0; 599 mw->buff_size = 0; 600 mw->virt_addr = NULL; 601 } 602 603 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, 604 unsigned int size) 605 { 606 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 607 struct pci_dev *pdev = nt->ndev->pdev; 608 unsigned int xlat_size, buff_size; 609 int rc; 610 611 xlat_size = round_up(size, mw->xlat_align_size); 612 buff_size = round_up(size, mw->xlat_align); 613 614 /* No need to re-setup */ 615 if (mw->xlat_size == xlat_size) 616 return 0; 617 618 if (mw->buff_size) 619 ntb_free_mw(nt, num_mw); 620 621 /* Alloc memory for receiving data. Must be aligned */ 622 mw->xlat_size = xlat_size; 623 mw->buff_size = buff_size; 624 625 mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size, 626 &mw->dma_addr, GFP_KERNEL); 627 if (!mw->virt_addr) { 628 mw->xlat_size = 0; 629 mw->buff_size = 0; 630 dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n", 631 buff_size); 632 return -ENOMEM; 633 } 634 635 /* 636 * we must ensure that the memory address allocated is BAR size 637 * aligned in order for the XLAT register to take the value. This 638 * is a requirement of the hardware. It is recommended to setup CMA 639 * for BAR sizes equal or greater than 4MB. 640 */ 641 if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) { 642 dev_err(&pdev->dev, "DMA memory %pad is not aligned\n", 643 &mw->dma_addr); 644 ntb_free_mw(nt, num_mw); 645 return -ENOMEM; 646 } 647 648 /* Notify HW the memory location of the receive buffer */ 649 rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size); 650 if (rc) { 651 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw); 652 ntb_free_mw(nt, num_mw); 653 return -EIO; 654 } 655 656 return 0; 657 } 658 659 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 660 { 661 qp->link_is_up = false; 662 663 qp->tx_index = 0; 664 qp->rx_index = 0; 665 qp->rx_bytes = 0; 666 qp->rx_pkts = 0; 667 qp->rx_ring_empty = 0; 668 qp->rx_err_no_buf = 0; 669 qp->rx_err_oflow = 0; 670 qp->rx_err_ver = 0; 671 qp->rx_memcpy = 0; 672 qp->rx_async = 0; 673 qp->tx_bytes = 0; 674 qp->tx_pkts = 0; 675 qp->tx_ring_full = 0; 676 qp->tx_err_no_buf = 0; 677 qp->tx_memcpy = 0; 678 qp->tx_async = 0; 679 } 680 681 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 682 { 683 struct ntb_transport_ctx *nt = qp->transport; 684 struct pci_dev *pdev = nt->ndev->pdev; 685 686 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num); 687 688 cancel_delayed_work_sync(&qp->link_work); 689 ntb_qp_link_down_reset(qp); 690 691 if (qp->event_handler) 692 qp->event_handler(qp->cb_data, qp->link_is_up); 693 } 694 695 static void ntb_qp_link_cleanup_work(struct work_struct *work) 696 { 697 struct ntb_transport_qp *qp = container_of(work, 698 struct ntb_transport_qp, 699 link_cleanup); 700 struct ntb_transport_ctx *nt = qp->transport; 701 702 ntb_qp_link_cleanup(qp); 703 704 if (nt->link_is_up) 705 schedule_delayed_work(&qp->link_work, 706 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 707 } 708 709 static void ntb_qp_link_down(struct ntb_transport_qp *qp) 710 { 711 schedule_work(&qp->link_cleanup); 712 } 713 714 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 715 { 716 struct ntb_transport_qp *qp; 717 u64 qp_bitmap_alloc; 718 int i; 719 720 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 721 722 /* Pass along the info to any clients */ 723 for (i = 0; i < nt->qp_count; i++) 724 if (qp_bitmap_alloc & BIT_ULL(i)) { 725 qp = &nt->qp_vec[i]; 726 ntb_qp_link_cleanup(qp); 727 cancel_work_sync(&qp->link_cleanup); 728 cancel_delayed_work_sync(&qp->link_work); 729 } 730 731 if (!nt->link_is_up) 732 cancel_delayed_work_sync(&nt->link_work); 733 734 /* The scratchpad registers keep the values if the remote side 735 * goes down, blast them now to give them a sane value the next 736 * time they are accessed 737 */ 738 for (i = 0; i < MAX_SPAD; i++) 739 ntb_spad_write(nt->ndev, i, 0); 740 } 741 742 static void ntb_transport_link_cleanup_work(struct work_struct *work) 743 { 744 struct ntb_transport_ctx *nt = 745 container_of(work, struct ntb_transport_ctx, link_cleanup); 746 747 ntb_transport_link_cleanup(nt); 748 } 749 750 static void ntb_transport_event_callback(void *data) 751 { 752 struct ntb_transport_ctx *nt = data; 753 754 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1) 755 schedule_delayed_work(&nt->link_work, 0); 756 else 757 schedule_work(&nt->link_cleanup); 758 } 759 760 static void ntb_transport_link_work(struct work_struct *work) 761 { 762 struct ntb_transport_ctx *nt = 763 container_of(work, struct ntb_transport_ctx, link_work.work); 764 struct ntb_dev *ndev = nt->ndev; 765 struct pci_dev *pdev = ndev->pdev; 766 resource_size_t size; 767 u32 val; 768 int rc, i, spad; 769 770 /* send the local info, in the opposite order of the way we read it */ 771 for (i = 0; i < nt->mw_count; i++) { 772 size = nt->mw_vec[i].phys_size; 773 774 if (max_mw_size && size > max_mw_size) 775 size = max_mw_size; 776 777 spad = MW0_SZ_HIGH + (i * 2); 778 ntb_peer_spad_write(ndev, spad, (u32)(size >> 32)); 779 780 spad = MW0_SZ_LOW + (i * 2); 781 ntb_peer_spad_write(ndev, spad, (u32)size); 782 } 783 784 ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count); 785 786 ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count); 787 788 ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION); 789 790 /* Query the remote side for its info */ 791 val = ntb_spad_read(ndev, VERSION); 792 dev_dbg(&pdev->dev, "Remote version = %d\n", val); 793 if (val != NTB_TRANSPORT_VERSION) 794 goto out; 795 796 val = ntb_spad_read(ndev, NUM_QPS); 797 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 798 if (val != nt->qp_count) 799 goto out; 800 801 val = ntb_spad_read(ndev, NUM_MWS); 802 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 803 if (val != nt->mw_count) 804 goto out; 805 806 for (i = 0; i < nt->mw_count; i++) { 807 u64 val64; 808 809 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2)); 810 val64 = (u64)val << 32; 811 812 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2)); 813 val64 |= val; 814 815 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64); 816 817 rc = ntb_set_mw(nt, i, val64); 818 if (rc) 819 goto out1; 820 } 821 822 nt->link_is_up = true; 823 824 for (i = 0; i < nt->qp_count; i++) { 825 struct ntb_transport_qp *qp = &nt->qp_vec[i]; 826 827 ntb_transport_setup_qp_mw(nt, i); 828 829 if (qp->client_ready) 830 schedule_delayed_work(&qp->link_work, 0); 831 } 832 833 return; 834 835 out1: 836 for (i = 0; i < nt->mw_count; i++) 837 ntb_free_mw(nt, i); 838 out: 839 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 840 schedule_delayed_work(&nt->link_work, 841 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 842 } 843 844 static void ntb_qp_link_work(struct work_struct *work) 845 { 846 struct ntb_transport_qp *qp = container_of(work, 847 struct ntb_transport_qp, 848 link_work.work); 849 struct pci_dev *pdev = qp->ndev->pdev; 850 struct ntb_transport_ctx *nt = qp->transport; 851 int val; 852 853 WARN_ON(!nt->link_is_up); 854 855 val = ntb_spad_read(nt->ndev, QP_LINKS); 856 857 ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num)); 858 859 /* query remote spad for qp ready bits */ 860 ntb_peer_spad_read(nt->ndev, QP_LINKS); 861 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val); 862 863 /* See if the remote side is up */ 864 if (val & BIT(qp->qp_num)) { 865 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 866 qp->link_is_up = true; 867 868 if (qp->event_handler) 869 qp->event_handler(qp->cb_data, qp->link_is_up); 870 } else if (nt->link_is_up) 871 schedule_delayed_work(&qp->link_work, 872 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 873 } 874 875 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt, 876 unsigned int qp_num) 877 { 878 struct ntb_transport_qp *qp; 879 struct ntb_transport_mw *mw; 880 phys_addr_t mw_base; 881 resource_size_t mw_size; 882 unsigned int num_qps_mw, tx_size; 883 unsigned int mw_num, mw_count, qp_count; 884 u64 qp_offset; 885 886 mw_count = nt->mw_count; 887 qp_count = nt->qp_count; 888 889 mw_num = QP_TO_MW(nt, qp_num); 890 mw = &nt->mw_vec[mw_num]; 891 892 qp = &nt->qp_vec[qp_num]; 893 qp->qp_num = qp_num; 894 qp->transport = nt; 895 qp->ndev = nt->ndev; 896 qp->client_ready = false; 897 qp->event_handler = NULL; 898 ntb_qp_link_down_reset(qp); 899 900 if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count) 901 num_qps_mw = qp_count / mw_count + 1; 902 else 903 num_qps_mw = qp_count / mw_count; 904 905 mw_base = nt->mw_vec[mw_num].phys_addr; 906 mw_size = nt->mw_vec[mw_num].phys_size; 907 908 tx_size = (unsigned int)mw_size / num_qps_mw; 909 qp_offset = tx_size * qp_num / mw_count; 910 911 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset; 912 if (!qp->tx_mw) 913 return -EINVAL; 914 915 qp->tx_mw_phys = mw_base + qp_offset; 916 if (!qp->tx_mw_phys) 917 return -EINVAL; 918 919 tx_size -= sizeof(struct ntb_rx_info); 920 qp->rx_info = qp->tx_mw + tx_size; 921 922 /* Due to housekeeping, there must be atleast 2 buffs */ 923 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 924 qp->tx_max_entry = tx_size / qp->tx_max_frame; 925 926 if (nt_debugfs_dir) { 927 char debugfs_name[4]; 928 929 snprintf(debugfs_name, 4, "qp%d", qp_num); 930 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 931 nt_debugfs_dir); 932 933 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 934 qp->debugfs_dir, qp, 935 &ntb_qp_debugfs_stats); 936 } else { 937 qp->debugfs_dir = NULL; 938 qp->debugfs_stats = NULL; 939 } 940 941 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 942 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); 943 944 spin_lock_init(&qp->ntb_rx_pend_q_lock); 945 spin_lock_init(&qp->ntb_rx_free_q_lock); 946 spin_lock_init(&qp->ntb_tx_free_q_lock); 947 948 INIT_LIST_HEAD(&qp->rx_pend_q); 949 INIT_LIST_HEAD(&qp->rx_free_q); 950 INIT_LIST_HEAD(&qp->tx_free_q); 951 952 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db, 953 (unsigned long)qp); 954 955 return 0; 956 } 957 958 static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev) 959 { 960 struct ntb_transport_ctx *nt; 961 struct ntb_transport_mw *mw; 962 unsigned int mw_count, qp_count; 963 u64 qp_bitmap; 964 int node; 965 int rc, i; 966 967 if (ntb_db_is_unsafe(ndev)) 968 dev_dbg(&ndev->dev, 969 "doorbell is unsafe, proceed anyway...\n"); 970 if (ntb_spad_is_unsafe(ndev)) 971 dev_dbg(&ndev->dev, 972 "scratchpad is unsafe, proceed anyway...\n"); 973 974 node = dev_to_node(&ndev->dev); 975 976 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node); 977 if (!nt) 978 return -ENOMEM; 979 980 nt->ndev = ndev; 981 982 mw_count = ntb_mw_count(ndev); 983 984 nt->mw_count = mw_count; 985 986 nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec), 987 GFP_KERNEL, node); 988 if (!nt->mw_vec) { 989 rc = -ENOMEM; 990 goto err; 991 } 992 993 for (i = 0; i < mw_count; i++) { 994 mw = &nt->mw_vec[i]; 995 996 rc = ntb_mw_get_range(ndev, i, &mw->phys_addr, &mw->phys_size, 997 &mw->xlat_align, &mw->xlat_align_size); 998 if (rc) 999 goto err1; 1000 1001 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1002 if (!mw->vbase) { 1003 rc = -ENOMEM; 1004 goto err1; 1005 } 1006 1007 mw->buff_size = 0; 1008 mw->xlat_size = 0; 1009 mw->virt_addr = NULL; 1010 mw->dma_addr = 0; 1011 } 1012 1013 qp_bitmap = ntb_db_valid_mask(ndev); 1014 1015 qp_count = ilog2(qp_bitmap); 1016 if (max_num_clients && max_num_clients < qp_count) 1017 qp_count = max_num_clients; 1018 else if (mw_count < qp_count) 1019 qp_count = mw_count; 1020 1021 qp_bitmap &= BIT_ULL(qp_count) - 1; 1022 1023 nt->qp_count = qp_count; 1024 nt->qp_bitmap = qp_bitmap; 1025 nt->qp_bitmap_free = qp_bitmap; 1026 1027 nt->qp_vec = kzalloc_node(qp_count * sizeof(*nt->qp_vec), 1028 GFP_KERNEL, node); 1029 if (!nt->qp_vec) { 1030 rc = -ENOMEM; 1031 goto err2; 1032 } 1033 1034 for (i = 0; i < qp_count; i++) { 1035 rc = ntb_transport_init_queue(nt, i); 1036 if (rc) 1037 goto err3; 1038 } 1039 1040 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1041 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1042 1043 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops); 1044 if (rc) 1045 goto err3; 1046 1047 INIT_LIST_HEAD(&nt->client_devs); 1048 rc = ntb_bus_init(nt); 1049 if (rc) 1050 goto err4; 1051 1052 nt->link_is_up = false; 1053 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1054 ntb_link_event(ndev); 1055 1056 return 0; 1057 1058 err4: 1059 ntb_clear_ctx(ndev); 1060 err3: 1061 kfree(nt->qp_vec); 1062 err2: 1063 kfree(nt->mw_vec); 1064 err1: 1065 while (i--) { 1066 mw = &nt->mw_vec[i]; 1067 iounmap(mw->vbase); 1068 } 1069 err: 1070 kfree(nt); 1071 return rc; 1072 } 1073 1074 static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev) 1075 { 1076 struct ntb_transport_ctx *nt = ndev->ctx; 1077 struct ntb_transport_qp *qp; 1078 u64 qp_bitmap_alloc; 1079 int i; 1080 1081 ntb_transport_link_cleanup(nt); 1082 cancel_work_sync(&nt->link_cleanup); 1083 cancel_delayed_work_sync(&nt->link_work); 1084 1085 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free; 1086 1087 /* verify that all the qp's are freed */ 1088 for (i = 0; i < nt->qp_count; i++) { 1089 qp = &nt->qp_vec[i]; 1090 if (qp_bitmap_alloc & BIT_ULL(i)) 1091 ntb_transport_free_queue(qp); 1092 debugfs_remove_recursive(qp->debugfs_dir); 1093 } 1094 1095 ntb_link_disable(ndev); 1096 ntb_clear_ctx(ndev); 1097 1098 ntb_bus_remove(nt); 1099 1100 for (i = nt->mw_count; i--; ) { 1101 ntb_free_mw(nt, i); 1102 iounmap(nt->mw_vec[i].vbase); 1103 } 1104 1105 kfree(nt->qp_vec); 1106 kfree(nt->mw_vec); 1107 kfree(nt); 1108 } 1109 1110 static void ntb_rx_copy_callback(void *data) 1111 { 1112 struct ntb_queue_entry *entry = data; 1113 struct ntb_transport_qp *qp = entry->qp; 1114 void *cb_data = entry->cb_data; 1115 unsigned int len = entry->len; 1116 struct ntb_payload_header *hdr = entry->rx_hdr; 1117 1118 hdr->flags = 0; 1119 1120 iowrite32(entry->index, &qp->rx_info->entry); 1121 1122 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); 1123 1124 if (qp->rx_handler && qp->client_ready) 1125 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1126 } 1127 1128 static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) 1129 { 1130 void *buf = entry->buf; 1131 size_t len = entry->len; 1132 1133 memcpy(buf, offset, len); 1134 1135 /* Ensure that the data is fully copied out before clearing the flag */ 1136 wmb(); 1137 1138 ntb_rx_copy_callback(entry); 1139 } 1140 1141 static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset, 1142 size_t len) 1143 { 1144 struct dma_async_tx_descriptor *txd; 1145 struct ntb_transport_qp *qp = entry->qp; 1146 struct dma_chan *chan = qp->dma_chan; 1147 struct dma_device *device; 1148 size_t pay_off, buff_off; 1149 struct dmaengine_unmap_data *unmap; 1150 dma_cookie_t cookie; 1151 void *buf = entry->buf; 1152 1153 entry->len = len; 1154 1155 if (!chan) 1156 goto err; 1157 1158 if (len < copy_bytes) 1159 goto err_wait; 1160 1161 device = chan->device; 1162 pay_off = (size_t)offset & ~PAGE_MASK; 1163 buff_off = (size_t)buf & ~PAGE_MASK; 1164 1165 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1166 goto err_wait; 1167 1168 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); 1169 if (!unmap) 1170 goto err_wait; 1171 1172 unmap->len = len; 1173 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), 1174 pay_off, len, DMA_TO_DEVICE); 1175 if (dma_mapping_error(device->dev, unmap->addr[0])) 1176 goto err_get_unmap; 1177 1178 unmap->to_cnt = 1; 1179 1180 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), 1181 buff_off, len, DMA_FROM_DEVICE); 1182 if (dma_mapping_error(device->dev, unmap->addr[1])) 1183 goto err_get_unmap; 1184 1185 unmap->from_cnt = 1; 1186 1187 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1188 unmap->addr[0], len, 1189 DMA_PREP_INTERRUPT); 1190 if (!txd) 1191 goto err_get_unmap; 1192 1193 txd->callback = ntb_rx_copy_callback; 1194 txd->callback_param = entry; 1195 dma_set_unmap(txd, unmap); 1196 1197 cookie = dmaengine_submit(txd); 1198 if (dma_submit_error(cookie)) 1199 goto err_set_unmap; 1200 1201 dmaengine_unmap_put(unmap); 1202 1203 qp->last_cookie = cookie; 1204 1205 qp->rx_async++; 1206 1207 return; 1208 1209 err_set_unmap: 1210 dmaengine_unmap_put(unmap); 1211 err_get_unmap: 1212 dmaengine_unmap_put(unmap); 1213 err_wait: 1214 /* If the callbacks come out of order, the writing of the index to the 1215 * last completed will be out of order. This may result in the 1216 * receive stalling forever. 1217 */ 1218 dma_sync_wait(chan, qp->last_cookie); 1219 err: 1220 ntb_memcpy_rx(entry, offset); 1221 qp->rx_memcpy++; 1222 } 1223 1224 static int ntb_process_rxc(struct ntb_transport_qp *qp) 1225 { 1226 struct ntb_payload_header *hdr; 1227 struct ntb_queue_entry *entry; 1228 void *offset; 1229 int rc; 1230 1231 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1232 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1233 1234 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n", 1235 qp->qp_num, hdr->ver, hdr->len, hdr->flags); 1236 1237 if (!(hdr->flags & DESC_DONE_FLAG)) { 1238 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n"); 1239 qp->rx_ring_empty++; 1240 return -EAGAIN; 1241 } 1242 1243 if (hdr->flags & LINK_DOWN_FLAG) { 1244 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n"); 1245 ntb_qp_link_down(qp); 1246 hdr->flags = 0; 1247 return -EAGAIN; 1248 } 1249 1250 if (hdr->ver != (u32)qp->rx_pkts) { 1251 dev_dbg(&qp->ndev->pdev->dev, 1252 "version mismatch, expected %llu - got %u\n", 1253 qp->rx_pkts, hdr->ver); 1254 qp->rx_err_ver++; 1255 return -EIO; 1256 } 1257 1258 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 1259 if (!entry) { 1260 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n"); 1261 qp->rx_err_no_buf++; 1262 1263 rc = -ENOMEM; 1264 goto err; 1265 } 1266 1267 if (hdr->len > entry->len) { 1268 dev_dbg(&qp->ndev->pdev->dev, 1269 "receive buffer overflow! Wanted %d got %d\n", 1270 hdr->len, entry->len); 1271 qp->rx_err_oflow++; 1272 1273 rc = -EIO; 1274 goto err; 1275 } 1276 1277 dev_dbg(&qp->ndev->pdev->dev, 1278 "RX OK index %u ver %u size %d into buf size %d\n", 1279 qp->rx_index, hdr->ver, hdr->len, entry->len); 1280 1281 qp->rx_bytes += hdr->len; 1282 qp->rx_pkts++; 1283 1284 entry->index = qp->rx_index; 1285 entry->rx_hdr = hdr; 1286 1287 ntb_async_rx(entry, offset, hdr->len); 1288 1289 qp->rx_index++; 1290 qp->rx_index %= qp->rx_max_entry; 1291 1292 return 0; 1293 1294 err: 1295 /* FIXME: if this syncrhonous update of the rx_index gets ahead of 1296 * asyncrhonous ntb_rx_copy_callback of previous entry, there are three 1297 * scenarios: 1298 * 1299 * 1) The peer might miss this update, but observe the update 1300 * from the memcpy completion callback. In this case, the buffer will 1301 * not be freed on the peer to be reused for a different packet. The 1302 * successful rx of a later packet would clear the condition, but the 1303 * condition could persist if several rx fail in a row. 1304 * 1305 * 2) The peer may observe this update before the asyncrhonous copy of 1306 * prior packets is completed. The peer may overwrite the buffers of 1307 * the prior packets before they are copied. 1308 * 1309 * 3) Both: the peer may observe the update, and then observe the index 1310 * decrement by the asynchronous completion callback. Who knows what 1311 * badness that will cause. 1312 */ 1313 hdr->flags = 0; 1314 iowrite32(qp->rx_index, &qp->rx_info->entry); 1315 1316 return rc; 1317 } 1318 1319 static void ntb_transport_rxc_db(unsigned long data) 1320 { 1321 struct ntb_transport_qp *qp = (void *)data; 1322 int rc, i; 1323 1324 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n", 1325 __func__, qp->qp_num); 1326 1327 /* Limit the number of packets processed in a single interrupt to 1328 * provide fairness to others 1329 */ 1330 for (i = 0; i < qp->rx_max_entry; i++) { 1331 rc = ntb_process_rxc(qp); 1332 if (rc) 1333 break; 1334 } 1335 1336 if (qp->dma_chan) 1337 dma_async_issue_pending(qp->dma_chan); 1338 1339 if (i == qp->rx_max_entry) { 1340 /* there is more work to do */ 1341 tasklet_schedule(&qp->rxc_db_work); 1342 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) { 1343 /* the doorbell bit is set: clear it */ 1344 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num)); 1345 /* ntb_db_read ensures ntb_db_clear write is committed */ 1346 ntb_db_read(qp->ndev); 1347 1348 /* an interrupt may have arrived between finishing 1349 * ntb_process_rxc and clearing the doorbell bit: 1350 * there might be some more work to do. 1351 */ 1352 tasklet_schedule(&qp->rxc_db_work); 1353 } 1354 } 1355 1356 static void ntb_tx_copy_callback(void *data) 1357 { 1358 struct ntb_queue_entry *entry = data; 1359 struct ntb_transport_qp *qp = entry->qp; 1360 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1361 1362 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1363 1364 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num)); 1365 1366 /* The entry length can only be zero if the packet is intended to be a 1367 * "link down" or similar. Since no payload is being sent in these 1368 * cases, there is nothing to add to the completion queue. 1369 */ 1370 if (entry->len > 0) { 1371 qp->tx_bytes += entry->len; 1372 1373 if (qp->tx_handler) 1374 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 1375 entry->len); 1376 } 1377 1378 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); 1379 } 1380 1381 static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) 1382 { 1383 #ifdef ARCH_HAS_NOCACHE_UACCESS 1384 /* 1385 * Using non-temporal mov to improve performance on non-cached 1386 * writes, even though we aren't actually copying from user space. 1387 */ 1388 __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); 1389 #else 1390 memcpy_toio(offset, entry->buf, entry->len); 1391 #endif 1392 1393 /* Ensure that the data is fully copied out before setting the flags */ 1394 wmb(); 1395 1396 ntb_tx_copy_callback(entry); 1397 } 1398 1399 static void ntb_async_tx(struct ntb_transport_qp *qp, 1400 struct ntb_queue_entry *entry) 1401 { 1402 struct ntb_payload_header __iomem *hdr; 1403 struct dma_async_tx_descriptor *txd; 1404 struct dma_chan *chan = qp->dma_chan; 1405 struct dma_device *device; 1406 size_t dest_off, buff_off; 1407 struct dmaengine_unmap_data *unmap; 1408 dma_addr_t dest; 1409 dma_cookie_t cookie; 1410 void __iomem *offset; 1411 size_t len = entry->len; 1412 void *buf = entry->buf; 1413 1414 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; 1415 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1416 entry->tx_hdr = hdr; 1417 1418 iowrite32(entry->len, &hdr->len); 1419 iowrite32((u32)qp->tx_pkts, &hdr->ver); 1420 1421 if (!chan) 1422 goto err; 1423 1424 if (len < copy_bytes) 1425 goto err; 1426 1427 device = chan->device; 1428 dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; 1429 buff_off = (size_t)buf & ~PAGE_MASK; 1430 dest_off = (size_t)dest & ~PAGE_MASK; 1431 1432 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1433 goto err; 1434 1435 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 1436 if (!unmap) 1437 goto err; 1438 1439 unmap->len = len; 1440 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), 1441 buff_off, len, DMA_TO_DEVICE); 1442 if (dma_mapping_error(device->dev, unmap->addr[0])) 1443 goto err_get_unmap; 1444 1445 unmap->to_cnt = 1; 1446 1447 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1448 DMA_PREP_INTERRUPT); 1449 if (!txd) 1450 goto err_get_unmap; 1451 1452 txd->callback = ntb_tx_copy_callback; 1453 txd->callback_param = entry; 1454 dma_set_unmap(txd, unmap); 1455 1456 cookie = dmaengine_submit(txd); 1457 if (dma_submit_error(cookie)) 1458 goto err_set_unmap; 1459 1460 dmaengine_unmap_put(unmap); 1461 1462 dma_async_issue_pending(chan); 1463 qp->tx_async++; 1464 1465 return; 1466 err_set_unmap: 1467 dmaengine_unmap_put(unmap); 1468 err_get_unmap: 1469 dmaengine_unmap_put(unmap); 1470 err: 1471 ntb_memcpy_tx(entry, offset); 1472 qp->tx_memcpy++; 1473 } 1474 1475 static int ntb_process_tx(struct ntb_transport_qp *qp, 1476 struct ntb_queue_entry *entry) 1477 { 1478 if (qp->tx_index == qp->remote_rx_info->entry) { 1479 qp->tx_ring_full++; 1480 return -EAGAIN; 1481 } 1482 1483 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1484 if (qp->tx_handler) 1485 qp->tx_handler(qp->cb_data, qp, NULL, -EIO); 1486 1487 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1488 &qp->tx_free_q); 1489 return 0; 1490 } 1491 1492 ntb_async_tx(qp, entry); 1493 1494 qp->tx_index++; 1495 qp->tx_index %= qp->tx_max_entry; 1496 1497 qp->tx_pkts++; 1498 1499 return 0; 1500 } 1501 1502 static void ntb_send_link_down(struct ntb_transport_qp *qp) 1503 { 1504 struct pci_dev *pdev = qp->ndev->pdev; 1505 struct ntb_queue_entry *entry; 1506 int i, rc; 1507 1508 if (!qp->link_is_up) 1509 return; 1510 1511 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num); 1512 1513 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1514 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1515 if (entry) 1516 break; 1517 msleep(100); 1518 } 1519 1520 if (!entry) 1521 return; 1522 1523 entry->cb_data = NULL; 1524 entry->buf = NULL; 1525 entry->len = 0; 1526 entry->flags = LINK_DOWN_FLAG; 1527 1528 rc = ntb_process_tx(qp, entry); 1529 if (rc) 1530 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", 1531 qp->qp_num); 1532 1533 ntb_qp_link_down_reset(qp); 1534 } 1535 1536 static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node) 1537 { 1538 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 1539 } 1540 1541 /** 1542 * ntb_transport_create_queue - Create a new NTB transport layer queue 1543 * @rx_handler: receive callback function 1544 * @tx_handler: transmit callback function 1545 * @event_handler: event callback function 1546 * 1547 * Create a new NTB transport layer queue and provide the queue with a callback 1548 * routine for both transmit and receive. The receive callback routine will be 1549 * used to pass up data when the transport has received it on the queue. The 1550 * transmit callback routine will be called when the transport has completed the 1551 * transmission of the data on the queue and the data is ready to be freed. 1552 * 1553 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1554 */ 1555 struct ntb_transport_qp * 1556 ntb_transport_create_queue(void *data, struct device *client_dev, 1557 const struct ntb_queue_handlers *handlers) 1558 { 1559 struct ntb_dev *ndev; 1560 struct pci_dev *pdev; 1561 struct ntb_transport_ctx *nt; 1562 struct ntb_queue_entry *entry; 1563 struct ntb_transport_qp *qp; 1564 u64 qp_bit; 1565 unsigned int free_queue; 1566 dma_cap_mask_t dma_mask; 1567 int node; 1568 int i; 1569 1570 ndev = dev_ntb(client_dev->parent); 1571 pdev = ndev->pdev; 1572 nt = ndev->ctx; 1573 1574 node = dev_to_node(&ndev->dev); 1575 1576 free_queue = ffs(nt->qp_bitmap); 1577 if (!free_queue) 1578 goto err; 1579 1580 /* decrement free_queue to make it zero based */ 1581 free_queue--; 1582 1583 qp = &nt->qp_vec[free_queue]; 1584 qp_bit = BIT_ULL(qp->qp_num); 1585 1586 nt->qp_bitmap_free &= ~qp_bit; 1587 1588 qp->cb_data = data; 1589 qp->rx_handler = handlers->rx_handler; 1590 qp->tx_handler = handlers->tx_handler; 1591 qp->event_handler = handlers->event_handler; 1592 1593 dma_cap_zero(dma_mask); 1594 dma_cap_set(DMA_MEMCPY, dma_mask); 1595 1596 if (use_dma) { 1597 qp->dma_chan = dma_request_channel(dma_mask, ntb_dma_filter_fn, 1598 (void *)(unsigned long)node); 1599 if (!qp->dma_chan) 1600 dev_info(&pdev->dev, "Unable to allocate DMA channel\n"); 1601 } else { 1602 qp->dma_chan = NULL; 1603 } 1604 dev_dbg(&pdev->dev, "Using %s memcpy\n", qp->dma_chan ? "DMA" : "CPU"); 1605 1606 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1607 entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); 1608 if (!entry) 1609 goto err1; 1610 1611 entry->qp = qp; 1612 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, 1613 &qp->rx_free_q); 1614 } 1615 1616 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1617 entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); 1618 if (!entry) 1619 goto err2; 1620 1621 entry->qp = qp; 1622 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1623 &qp->tx_free_q); 1624 } 1625 1626 ntb_db_clear(qp->ndev, qp_bit); 1627 ntb_db_clear_mask(qp->ndev, qp_bit); 1628 1629 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 1630 1631 return qp; 1632 1633 err2: 1634 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1635 kfree(entry); 1636 err1: 1637 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 1638 kfree(entry); 1639 if (qp->dma_chan) 1640 dma_release_channel(qp->dma_chan); 1641 nt->qp_bitmap_free |= qp_bit; 1642 err: 1643 return NULL; 1644 } 1645 EXPORT_SYMBOL_GPL(ntb_transport_create_queue); 1646 1647 /** 1648 * ntb_transport_free_queue - Frees NTB transport queue 1649 * @qp: NTB queue to be freed 1650 * 1651 * Frees NTB transport queue 1652 */ 1653 void ntb_transport_free_queue(struct ntb_transport_qp *qp) 1654 { 1655 struct ntb_transport_ctx *nt = qp->transport; 1656 struct pci_dev *pdev; 1657 struct ntb_queue_entry *entry; 1658 u64 qp_bit; 1659 1660 if (!qp) 1661 return; 1662 1663 pdev = qp->ndev->pdev; 1664 1665 if (qp->dma_chan) { 1666 struct dma_chan *chan = qp->dma_chan; 1667 /* Putting the dma_chan to NULL will force any new traffic to be 1668 * processed by the CPU instead of the DAM engine 1669 */ 1670 qp->dma_chan = NULL; 1671 1672 /* Try to be nice and wait for any queued DMA engine 1673 * transactions to process before smashing it with a rock 1674 */ 1675 dma_sync_wait(chan, qp->last_cookie); 1676 dmaengine_terminate_all(chan); 1677 dma_release_channel(chan); 1678 } 1679 1680 qp_bit = BIT_ULL(qp->qp_num); 1681 1682 ntb_db_set_mask(qp->ndev, qp_bit); 1683 tasklet_disable(&qp->rxc_db_work); 1684 1685 cancel_delayed_work_sync(&qp->link_work); 1686 1687 qp->cb_data = NULL; 1688 qp->rx_handler = NULL; 1689 qp->tx_handler = NULL; 1690 qp->event_handler = NULL; 1691 1692 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 1693 kfree(entry); 1694 1695 while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) { 1696 dev_warn(&pdev->dev, "Freeing item from a non-empty queue\n"); 1697 kfree(entry); 1698 } 1699 1700 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1701 kfree(entry); 1702 1703 nt->qp_bitmap_free |= qp_bit; 1704 1705 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 1706 } 1707 EXPORT_SYMBOL_GPL(ntb_transport_free_queue); 1708 1709 /** 1710 * ntb_transport_rx_remove - Dequeues enqueued rx packet 1711 * @qp: NTB queue to be freed 1712 * @len: pointer to variable to write enqueued buffers length 1713 * 1714 * Dequeues unused buffers from receive queue. Should only be used during 1715 * shutdown of qp. 1716 * 1717 * RETURNS: NULL error value on error, or void* for success. 1718 */ 1719 void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) 1720 { 1721 struct ntb_queue_entry *entry; 1722 void *buf; 1723 1724 if (!qp || qp->client_ready) 1725 return NULL; 1726 1727 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 1728 if (!entry) 1729 return NULL; 1730 1731 buf = entry->cb_data; 1732 *len = entry->len; 1733 1734 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); 1735 1736 return buf; 1737 } 1738 EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); 1739 1740 /** 1741 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry 1742 * @qp: NTB transport layer queue the entry is to be enqueued on 1743 * @cb: per buffer pointer for callback function to use 1744 * @data: pointer to data buffer that incoming packets will be copied into 1745 * @len: length of the data buffer 1746 * 1747 * Enqueue a new receive buffer onto the transport queue into which a NTB 1748 * payload can be received into. 1749 * 1750 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1751 */ 1752 int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 1753 unsigned int len) 1754 { 1755 struct ntb_queue_entry *entry; 1756 1757 if (!qp) 1758 return -EINVAL; 1759 1760 entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q); 1761 if (!entry) 1762 return -ENOMEM; 1763 1764 entry->cb_data = cb; 1765 entry->buf = data; 1766 entry->len = len; 1767 1768 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); 1769 1770 return 0; 1771 } 1772 EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); 1773 1774 /** 1775 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 1776 * @qp: NTB transport layer queue the entry is to be enqueued on 1777 * @cb: per buffer pointer for callback function to use 1778 * @data: pointer to data buffer that will be sent 1779 * @len: length of the data buffer 1780 * 1781 * Enqueue a new transmit buffer onto the transport queue from which a NTB 1782 * payload will be transmitted. This assumes that a lock is being held to 1783 * serialize access to the qp. 1784 * 1785 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 1786 */ 1787 int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 1788 unsigned int len) 1789 { 1790 struct ntb_queue_entry *entry; 1791 int rc; 1792 1793 if (!qp || !qp->link_is_up || !len) 1794 return -EINVAL; 1795 1796 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1797 if (!entry) { 1798 qp->tx_err_no_buf++; 1799 return -ENOMEM; 1800 } 1801 1802 entry->cb_data = cb; 1803 entry->buf = data; 1804 entry->len = len; 1805 entry->flags = 0; 1806 1807 rc = ntb_process_tx(qp, entry); 1808 if (rc) 1809 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1810 &qp->tx_free_q); 1811 1812 return rc; 1813 } 1814 EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); 1815 1816 /** 1817 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 1818 * @qp: NTB transport layer queue to be enabled 1819 * 1820 * Notify NTB transport layer of client readiness to use queue 1821 */ 1822 void ntb_transport_link_up(struct ntb_transport_qp *qp) 1823 { 1824 if (!qp) 1825 return; 1826 1827 qp->client_ready = true; 1828 1829 if (qp->transport->link_is_up) 1830 schedule_delayed_work(&qp->link_work, 0); 1831 } 1832 EXPORT_SYMBOL_GPL(ntb_transport_link_up); 1833 1834 /** 1835 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1836 * @qp: NTB transport layer queue to be disabled 1837 * 1838 * Notify NTB transport layer of client's desire to no longer receive data on 1839 * transport queue specified. It is the client's responsibility to ensure all 1840 * entries on queue are purged or otherwise handled appropriately. 1841 */ 1842 void ntb_transport_link_down(struct ntb_transport_qp *qp) 1843 { 1844 struct pci_dev *pdev; 1845 int val; 1846 1847 if (!qp) 1848 return; 1849 1850 pdev = qp->ndev->pdev; 1851 qp->client_ready = false; 1852 1853 val = ntb_spad_read(qp->ndev, QP_LINKS); 1854 1855 ntb_peer_spad_write(qp->ndev, QP_LINKS, 1856 val & ~BIT(qp->qp_num)); 1857 1858 if (qp->link_is_up) 1859 ntb_send_link_down(qp); 1860 else 1861 cancel_delayed_work_sync(&qp->link_work); 1862 } 1863 EXPORT_SYMBOL_GPL(ntb_transport_link_down); 1864 1865 /** 1866 * ntb_transport_link_query - Query transport link state 1867 * @qp: NTB transport layer queue to be queried 1868 * 1869 * Query connectivity to the remote system of the NTB transport queue 1870 * 1871 * RETURNS: true for link up or false for link down 1872 */ 1873 bool ntb_transport_link_query(struct ntb_transport_qp *qp) 1874 { 1875 if (!qp) 1876 return false; 1877 1878 return qp->link_is_up; 1879 } 1880 EXPORT_SYMBOL_GPL(ntb_transport_link_query); 1881 1882 /** 1883 * ntb_transport_qp_num - Query the qp number 1884 * @qp: NTB transport layer queue to be queried 1885 * 1886 * Query qp number of the NTB transport queue 1887 * 1888 * RETURNS: a zero based number specifying the qp number 1889 */ 1890 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) 1891 { 1892 if (!qp) 1893 return 0; 1894 1895 return qp->qp_num; 1896 } 1897 EXPORT_SYMBOL_GPL(ntb_transport_qp_num); 1898 1899 /** 1900 * ntb_transport_max_size - Query the max payload size of a qp 1901 * @qp: NTB transport layer queue to be queried 1902 * 1903 * Query the maximum payload size permissible on the given qp 1904 * 1905 * RETURNS: the max payload size of a qp 1906 */ 1907 unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) 1908 { 1909 unsigned int max; 1910 1911 if (!qp) 1912 return 0; 1913 1914 if (!qp->dma_chan) 1915 return qp->tx_max_frame - sizeof(struct ntb_payload_header); 1916 1917 /* If DMA engine usage is possible, try to find the max size for that */ 1918 max = qp->tx_max_frame - sizeof(struct ntb_payload_header); 1919 max -= max % (1 << qp->dma_chan->device->copy_align); 1920 1921 return max; 1922 } 1923 EXPORT_SYMBOL_GPL(ntb_transport_max_size); 1924 1925 static void ntb_transport_doorbell_callback(void *data, int vector) 1926 { 1927 struct ntb_transport_ctx *nt = data; 1928 struct ntb_transport_qp *qp; 1929 u64 db_bits; 1930 unsigned int qp_num; 1931 1932 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free & 1933 ntb_db_vector_mask(nt->ndev, vector)); 1934 1935 while (db_bits) { 1936 qp_num = __ffs(db_bits); 1937 qp = &nt->qp_vec[qp_num]; 1938 1939 tasklet_schedule(&qp->rxc_db_work); 1940 1941 db_bits &= ~BIT_ULL(qp_num); 1942 } 1943 } 1944 1945 static const struct ntb_ctx_ops ntb_transport_ops = { 1946 .link_event = ntb_transport_event_callback, 1947 .db_event = ntb_transport_doorbell_callback, 1948 }; 1949 1950 static struct ntb_client ntb_transport_client = { 1951 .ops = { 1952 .probe = ntb_transport_probe, 1953 .remove = ntb_transport_free, 1954 }, 1955 }; 1956 1957 static int __init ntb_transport_init(void) 1958 { 1959 int rc; 1960 1961 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER); 1962 1963 if (debugfs_initialized()) 1964 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1965 1966 rc = bus_register(&ntb_transport_bus); 1967 if (rc) 1968 goto err_bus; 1969 1970 rc = ntb_register_client(&ntb_transport_client); 1971 if (rc) 1972 goto err_client; 1973 1974 return 0; 1975 1976 err_client: 1977 bus_unregister(&ntb_transport_bus); 1978 err_bus: 1979 debugfs_remove_recursive(nt_debugfs_dir); 1980 return rc; 1981 } 1982 module_init(ntb_transport_init); 1983 1984 static void __exit ntb_transport_exit(void) 1985 { 1986 debugfs_remove_recursive(nt_debugfs_dir); 1987 1988 ntb_unregister_client(&ntb_transport_client); 1989 bus_unregister(&ntb_transport_bus); 1990 } 1991 module_exit(ntb_transport_exit); 1992