1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2015 Intel Corporation. All rights reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * BSD LICENSE 14 * 15 * Copyright(c) 2015 Intel Corporation. All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 21 * * Redistributions of source code must retain the above copyright 22 * notice, this list of conditions and the following disclaimer. 23 * * Redistributions in binary form must reproduce the above copy 24 * notice, this list of conditions and the following disclaimer in 25 * the documentation and/or other materials provided with the 26 * distribution. 27 * * Neither the name of Intel Corporation nor the names of its 28 * contributors may be used to endorse or promote products derived 29 * from this software without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 * 43 * PCIe NTB Perf Linux driver 44 */ 45 46 #include <linux/init.h> 47 #include <linux/kernel.h> 48 #include <linux/module.h> 49 #include <linux/kthread.h> 50 #include <linux/time.h> 51 #include <linux/timer.h> 52 #include <linux/dma-mapping.h> 53 #include <linux/pci.h> 54 #include <linux/slab.h> 55 #include <linux/spinlock.h> 56 #include <linux/debugfs.h> 57 #include <linux/dmaengine.h> 58 #include <linux/delay.h> 59 #include <linux/sizes.h> 60 #include <linux/ntb.h> 61 #include <linux/mutex.h> 62 63 #define DRIVER_NAME "ntb_perf" 64 #define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool" 65 66 #define DRIVER_VERSION "1.0" 67 #define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>" 68 69 #define PERF_LINK_DOWN_TIMEOUT 10 70 #define PERF_VERSION 0xffff0001 71 #define MAX_THREADS 32 72 #define MAX_TEST_SIZE SZ_1M 73 #define MAX_SRCS 32 74 #define DMA_OUT_RESOURCE_TO msecs_to_jiffies(50) 75 #define DMA_RETRIES 20 76 #define SZ_4G (1ULL << 32) 77 #define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */ 78 #define PIDX NTB_DEF_PEER_IDX 79 80 MODULE_LICENSE("Dual BSD/GPL"); 81 MODULE_VERSION(DRIVER_VERSION); 82 MODULE_AUTHOR(DRIVER_AUTHOR); 83 MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 84 85 static struct dentry *perf_debugfs_dir; 86 87 static unsigned long max_mw_size; 88 module_param(max_mw_size, ulong, 0644); 89 MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 90 91 static unsigned int seg_order = 19; /* 512K */ 92 module_param(seg_order, uint, 0644); 93 MODULE_PARM_DESC(seg_order, "size order [2^n] of buffer segment for testing"); 94 95 static unsigned int run_order = 32; /* 4G */ 96 module_param(run_order, uint, 0644); 97 MODULE_PARM_DESC(run_order, "size order [2^n] of total data to transfer"); 98 99 static bool use_dma; /* default to 0 */ 100 module_param(use_dma, bool, 0644); 101 MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance"); 102 103 static bool on_node = true; /* default to 1 */ 104 module_param(on_node, bool, 0644); 105 MODULE_PARM_DESC(on_node, "Run threads only on NTB device node (default: true)"); 106 107 struct perf_mw { 108 phys_addr_t phys_addr; 109 resource_size_t phys_size; 110 void __iomem *vbase; 111 size_t xlat_size; 112 size_t buf_size; 113 void *virt_addr; 114 dma_addr_t dma_addr; 115 }; 116 117 struct perf_ctx; 118 119 struct pthr_ctx { 120 struct task_struct *thread; 121 struct perf_ctx *perf; 122 atomic_t dma_sync; 123 struct dma_chan *dma_chan; 124 int dma_prep_err; 125 int src_idx; 126 void *srcs[MAX_SRCS]; 127 wait_queue_head_t *wq; 128 int status; 129 u64 copied; 130 u64 diff_us; 131 }; 132 133 struct perf_ctx { 134 struct ntb_dev *ntb; 135 spinlock_t db_lock; 136 struct perf_mw mw; 137 bool link_is_up; 138 struct delayed_work link_work; 139 wait_queue_head_t link_wq; 140 u8 perf_threads; 141 /* mutex ensures only one set of threads run at once */ 142 struct mutex run_mutex; 143 struct pthr_ctx pthr_ctx[MAX_THREADS]; 144 atomic_t tsync; 145 atomic_t tdone; 146 }; 147 148 enum { 149 VERSION = 0, 150 MW_SZ_HIGH, 151 MW_SZ_LOW, 152 MAX_SPAD 153 }; 154 155 static void perf_link_event(void *ctx) 156 { 157 struct perf_ctx *perf = ctx; 158 159 if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) { 160 schedule_delayed_work(&perf->link_work, 2*HZ); 161 } else { 162 dev_dbg(&perf->ntb->pdev->dev, "link down\n"); 163 164 if (!perf->link_is_up) 165 cancel_delayed_work_sync(&perf->link_work); 166 167 perf->link_is_up = false; 168 } 169 } 170 171 static void perf_db_event(void *ctx, int vec) 172 { 173 struct perf_ctx *perf = ctx; 174 u64 db_bits, db_mask; 175 176 db_mask = ntb_db_vector_mask(perf->ntb, vec); 177 db_bits = ntb_db_read(perf->ntb); 178 179 dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", 180 vec, db_mask, db_bits); 181 } 182 183 static const struct ntb_ctx_ops perf_ops = { 184 .link_event = perf_link_event, 185 .db_event = perf_db_event, 186 }; 187 188 static void perf_copy_callback(void *data) 189 { 190 struct pthr_ctx *pctx = data; 191 192 atomic_dec(&pctx->dma_sync); 193 } 194 195 static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst, 196 char *src, size_t size) 197 { 198 struct perf_ctx *perf = pctx->perf; 199 struct dma_async_tx_descriptor *txd; 200 struct dma_chan *chan = pctx->dma_chan; 201 struct dma_device *device; 202 struct dmaengine_unmap_data *unmap; 203 dma_cookie_t cookie; 204 size_t src_off, dst_off; 205 struct perf_mw *mw = &perf->mw; 206 void __iomem *vbase; 207 void __iomem *dst_vaddr; 208 dma_addr_t dst_phys; 209 int retries = 0; 210 211 if (!use_dma) { 212 memcpy_toio(dst, src, size); 213 return size; 214 } 215 216 if (!chan) { 217 dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); 218 return -EINVAL; 219 } 220 221 device = chan->device; 222 src_off = (uintptr_t)src & ~PAGE_MASK; 223 dst_off = (uintptr_t __force)dst & ~PAGE_MASK; 224 225 if (!is_dma_copy_aligned(device, src_off, dst_off, size)) 226 return -ENODEV; 227 228 vbase = mw->vbase; 229 dst_vaddr = dst; 230 dst_phys = mw->phys_addr + (dst_vaddr - vbase); 231 232 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 233 if (!unmap) 234 return -ENOMEM; 235 236 unmap->len = size; 237 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), 238 src_off, size, DMA_TO_DEVICE); 239 if (dma_mapping_error(device->dev, unmap->addr[0])) 240 goto err_get_unmap; 241 242 unmap->to_cnt = 1; 243 244 do { 245 txd = device->device_prep_dma_memcpy(chan, dst_phys, 246 unmap->addr[0], 247 size, DMA_PREP_INTERRUPT); 248 if (!txd) { 249 set_current_state(TASK_INTERRUPTIBLE); 250 schedule_timeout(DMA_OUT_RESOURCE_TO); 251 } 252 } while (!txd && (++retries < DMA_RETRIES)); 253 254 if (!txd) { 255 pctx->dma_prep_err++; 256 goto err_get_unmap; 257 } 258 259 txd->callback = perf_copy_callback; 260 txd->callback_param = pctx; 261 dma_set_unmap(txd, unmap); 262 263 cookie = dmaengine_submit(txd); 264 if (dma_submit_error(cookie)) 265 goto err_set_unmap; 266 267 dmaengine_unmap_put(unmap); 268 269 atomic_inc(&pctx->dma_sync); 270 dma_async_issue_pending(chan); 271 272 return size; 273 274 err_set_unmap: 275 dmaengine_unmap_put(unmap); 276 err_get_unmap: 277 dmaengine_unmap_put(unmap); 278 return 0; 279 } 280 281 static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src, 282 u64 buf_size, u64 win_size, u64 total) 283 { 284 int chunks, total_chunks, i; 285 int copied_chunks = 0; 286 u64 copied = 0, result; 287 char __iomem *tmp = dst; 288 u64 perf, diff_us; 289 ktime_t kstart, kstop, kdiff; 290 unsigned long last_sleep = jiffies; 291 292 chunks = div64_u64(win_size, buf_size); 293 total_chunks = div64_u64(total, buf_size); 294 kstart = ktime_get(); 295 296 for (i = 0; i < total_chunks; i++) { 297 result = perf_copy(pctx, tmp, src, buf_size); 298 copied += result; 299 copied_chunks++; 300 if (copied_chunks == chunks) { 301 tmp = dst; 302 copied_chunks = 0; 303 } else 304 tmp += buf_size; 305 306 /* Probably should schedule every 5s to prevent soft hang. */ 307 if (unlikely((jiffies - last_sleep) > 5 * HZ)) { 308 last_sleep = jiffies; 309 set_current_state(TASK_INTERRUPTIBLE); 310 schedule_timeout(1); 311 } 312 313 if (unlikely(kthread_should_stop())) 314 break; 315 } 316 317 if (use_dma) { 318 pr_debug("%s: All DMA descriptors submitted\n", current->comm); 319 while (atomic_read(&pctx->dma_sync) != 0) { 320 if (kthread_should_stop()) 321 break; 322 msleep(20); 323 } 324 } 325 326 kstop = ktime_get(); 327 kdiff = ktime_sub(kstop, kstart); 328 diff_us = ktime_to_us(kdiff); 329 330 pr_debug("%s: copied %llu bytes\n", current->comm, copied); 331 332 pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us); 333 334 perf = div64_u64(copied, diff_us); 335 336 pr_debug("%s: MBytes/s: %llu\n", current->comm, perf); 337 338 pctx->copied = copied; 339 pctx->diff_us = diff_us; 340 341 return 0; 342 } 343 344 static bool perf_dma_filter_fn(struct dma_chan *chan, void *node) 345 { 346 /* Is the channel required to be on the same node as the device? */ 347 if (!on_node) 348 return true; 349 350 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 351 } 352 353 static int ntb_perf_thread(void *data) 354 { 355 struct pthr_ctx *pctx = data; 356 struct perf_ctx *perf = pctx->perf; 357 struct pci_dev *pdev = perf->ntb->pdev; 358 struct perf_mw *mw = &perf->mw; 359 char __iomem *dst; 360 u64 win_size, buf_size, total; 361 void *src; 362 int rc, node, i; 363 struct dma_chan *dma_chan = NULL; 364 365 pr_debug("kthread %s starting...\n", current->comm); 366 367 node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE; 368 369 if (use_dma && !pctx->dma_chan) { 370 dma_cap_mask_t dma_mask; 371 372 dma_cap_zero(dma_mask); 373 dma_cap_set(DMA_MEMCPY, dma_mask); 374 dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn, 375 (void *)(unsigned long)node); 376 if (!dma_chan) { 377 pr_warn("%s: cannot acquire DMA channel, quitting\n", 378 current->comm); 379 return -ENODEV; 380 } 381 pctx->dma_chan = dma_chan; 382 } 383 384 for (i = 0; i < MAX_SRCS; i++) { 385 pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node); 386 if (!pctx->srcs[i]) { 387 rc = -ENOMEM; 388 goto err; 389 } 390 } 391 392 win_size = mw->phys_size; 393 buf_size = 1ULL << seg_order; 394 total = 1ULL << run_order; 395 396 if (buf_size > MAX_TEST_SIZE) 397 buf_size = MAX_TEST_SIZE; 398 399 dst = (char __iomem *)mw->vbase; 400 401 atomic_inc(&perf->tsync); 402 while (atomic_read(&perf->tsync) != perf->perf_threads) 403 schedule(); 404 405 src = pctx->srcs[pctx->src_idx]; 406 pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1); 407 408 rc = perf_move_data(pctx, dst, src, buf_size, win_size, total); 409 410 atomic_dec(&perf->tsync); 411 412 if (rc < 0) { 413 pr_err("%s: failed\n", current->comm); 414 rc = -ENXIO; 415 goto err; 416 } 417 418 for (i = 0; i < MAX_SRCS; i++) { 419 kfree(pctx->srcs[i]); 420 pctx->srcs[i] = NULL; 421 } 422 423 atomic_inc(&perf->tdone); 424 wake_up(pctx->wq); 425 rc = 0; 426 goto done; 427 428 err: 429 for (i = 0; i < MAX_SRCS; i++) { 430 kfree(pctx->srcs[i]); 431 pctx->srcs[i] = NULL; 432 } 433 434 if (dma_chan) { 435 dma_release_channel(dma_chan); 436 pctx->dma_chan = NULL; 437 } 438 439 done: 440 /* Wait until we are told to stop */ 441 for (;;) { 442 set_current_state(TASK_INTERRUPTIBLE); 443 if (kthread_should_stop()) 444 break; 445 schedule(); 446 } 447 __set_current_state(TASK_RUNNING); 448 449 return rc; 450 } 451 452 static void perf_free_mw(struct perf_ctx *perf) 453 { 454 struct perf_mw *mw = &perf->mw; 455 struct pci_dev *pdev = perf->ntb->pdev; 456 457 if (!mw->virt_addr) 458 return; 459 460 ntb_mw_clear_trans(perf->ntb, PIDX, 0); 461 dma_free_coherent(&pdev->dev, mw->buf_size, 462 mw->virt_addr, mw->dma_addr); 463 mw->xlat_size = 0; 464 mw->buf_size = 0; 465 mw->virt_addr = NULL; 466 } 467 468 static int perf_set_mw(struct perf_ctx *perf, resource_size_t size) 469 { 470 struct perf_mw *mw = &perf->mw; 471 size_t xlat_size, buf_size; 472 resource_size_t xlat_align; 473 resource_size_t xlat_align_size; 474 int rc; 475 476 if (!size) 477 return -EINVAL; 478 479 rc = ntb_mw_get_align(perf->ntb, PIDX, 0, &xlat_align, 480 &xlat_align_size, NULL); 481 if (rc) 482 return rc; 483 484 xlat_size = round_up(size, xlat_align_size); 485 buf_size = round_up(size, xlat_align); 486 487 if (mw->xlat_size == xlat_size) 488 return 0; 489 490 if (mw->buf_size) 491 perf_free_mw(perf); 492 493 mw->xlat_size = xlat_size; 494 mw->buf_size = buf_size; 495 496 mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size, 497 &mw->dma_addr, GFP_KERNEL); 498 if (!mw->virt_addr) { 499 mw->xlat_size = 0; 500 mw->buf_size = 0; 501 } 502 503 rc = ntb_mw_set_trans(perf->ntb, PIDX, 0, mw->dma_addr, mw->xlat_size); 504 if (rc) { 505 dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n"); 506 perf_free_mw(perf); 507 return -EIO; 508 } 509 510 return 0; 511 } 512 513 static void perf_link_work(struct work_struct *work) 514 { 515 struct perf_ctx *perf = 516 container_of(work, struct perf_ctx, link_work.work); 517 struct ntb_dev *ndev = perf->ntb; 518 struct pci_dev *pdev = ndev->pdev; 519 u32 val; 520 u64 size; 521 int rc; 522 523 dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); 524 525 size = perf->mw.phys_size; 526 527 if (max_mw_size && size > max_mw_size) 528 size = max_mw_size; 529 530 ntb_peer_spad_write(ndev, PIDX, MW_SZ_HIGH, upper_32_bits(size)); 531 ntb_peer_spad_write(ndev, PIDX, MW_SZ_LOW, lower_32_bits(size)); 532 ntb_peer_spad_write(ndev, PIDX, VERSION, PERF_VERSION); 533 534 /* now read what peer wrote */ 535 val = ntb_spad_read(ndev, VERSION); 536 if (val != PERF_VERSION) { 537 dev_dbg(&pdev->dev, "Remote version = %#x\n", val); 538 goto out; 539 } 540 541 val = ntb_spad_read(ndev, MW_SZ_HIGH); 542 size = (u64)val << 32; 543 544 val = ntb_spad_read(ndev, MW_SZ_LOW); 545 size |= val; 546 547 dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size); 548 549 rc = perf_set_mw(perf, size); 550 if (rc) 551 goto out1; 552 553 perf->link_is_up = true; 554 wake_up(&perf->link_wq); 555 556 return; 557 558 out1: 559 perf_free_mw(perf); 560 561 out: 562 if (ntb_link_is_up(ndev, NULL, NULL) == 1) 563 schedule_delayed_work(&perf->link_work, 564 msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT)); 565 } 566 567 static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf) 568 { 569 struct perf_mw *mw; 570 int rc; 571 572 mw = &perf->mw; 573 574 rc = ntb_peer_mw_get_addr(ntb, 0, &mw->phys_addr, &mw->phys_size); 575 if (rc) 576 return rc; 577 578 perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 579 if (!mw->vbase) 580 return -ENOMEM; 581 582 return 0; 583 } 584 585 static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf, 586 size_t count, loff_t *offp) 587 { 588 struct perf_ctx *perf = filp->private_data; 589 char *buf; 590 ssize_t ret, out_off = 0; 591 struct pthr_ctx *pctx; 592 int i; 593 u64 rate; 594 595 if (!perf) 596 return 0; 597 598 buf = kmalloc(1024, GFP_KERNEL); 599 if (!buf) 600 return -ENOMEM; 601 602 if (mutex_is_locked(&perf->run_mutex)) { 603 out_off = scnprintf(buf, 64, "running\n"); 604 goto read_from_buf; 605 } 606 607 for (i = 0; i < MAX_THREADS; i++) { 608 pctx = &perf->pthr_ctx[i]; 609 610 if (pctx->status == -ENODATA) 611 break; 612 613 if (pctx->status) { 614 out_off += scnprintf(buf + out_off, 1024 - out_off, 615 "%d: error %d\n", i, 616 pctx->status); 617 continue; 618 } 619 620 rate = div64_u64(pctx->copied, pctx->diff_us); 621 out_off += scnprintf(buf + out_off, 1024 - out_off, 622 "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n", 623 i, pctx->copied, pctx->diff_us, rate); 624 } 625 626 read_from_buf: 627 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off); 628 kfree(buf); 629 630 return ret; 631 } 632 633 static void threads_cleanup(struct perf_ctx *perf) 634 { 635 struct pthr_ctx *pctx; 636 int i; 637 638 for (i = 0; i < MAX_THREADS; i++) { 639 pctx = &perf->pthr_ctx[i]; 640 if (pctx->thread) { 641 pctx->status = kthread_stop(pctx->thread); 642 pctx->thread = NULL; 643 } 644 } 645 } 646 647 static void perf_clear_thread_status(struct perf_ctx *perf) 648 { 649 int i; 650 651 for (i = 0; i < MAX_THREADS; i++) 652 perf->pthr_ctx[i].status = -ENODATA; 653 } 654 655 static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf, 656 size_t count, loff_t *offp) 657 { 658 struct perf_ctx *perf = filp->private_data; 659 int node, i; 660 DECLARE_WAIT_QUEUE_HEAD(wq); 661 662 if (wait_event_interruptible(perf->link_wq, perf->link_is_up)) 663 return -ENOLINK; 664 665 if (perf->perf_threads == 0) 666 return -EINVAL; 667 668 if (!mutex_trylock(&perf->run_mutex)) 669 return -EBUSY; 670 671 perf_clear_thread_status(perf); 672 673 if (perf->perf_threads > MAX_THREADS) { 674 perf->perf_threads = MAX_THREADS; 675 pr_info("Reset total threads to: %u\n", MAX_THREADS); 676 } 677 678 /* no greater than 1M */ 679 if (seg_order > MAX_SEG_ORDER) { 680 seg_order = MAX_SEG_ORDER; 681 pr_info("Fix seg_order to %u\n", seg_order); 682 } 683 684 if (run_order < seg_order) { 685 run_order = seg_order; 686 pr_info("Fix run_order to %u\n", run_order); 687 } 688 689 node = on_node ? dev_to_node(&perf->ntb->pdev->dev) 690 : NUMA_NO_NODE; 691 atomic_set(&perf->tdone, 0); 692 693 /* launch kernel thread */ 694 for (i = 0; i < perf->perf_threads; i++) { 695 struct pthr_ctx *pctx; 696 697 pctx = &perf->pthr_ctx[i]; 698 atomic_set(&pctx->dma_sync, 0); 699 pctx->perf = perf; 700 pctx->wq = &wq; 701 pctx->thread = 702 kthread_create_on_node(ntb_perf_thread, 703 (void *)pctx, 704 node, "ntb_perf %d", i); 705 if (IS_ERR(pctx->thread)) { 706 pctx->thread = NULL; 707 goto err; 708 } else { 709 wake_up_process(pctx->thread); 710 } 711 } 712 713 wait_event_interruptible(wq, 714 atomic_read(&perf->tdone) == perf->perf_threads); 715 716 threads_cleanup(perf); 717 mutex_unlock(&perf->run_mutex); 718 return count; 719 720 err: 721 threads_cleanup(perf); 722 mutex_unlock(&perf->run_mutex); 723 return -ENXIO; 724 } 725 726 static const struct file_operations ntb_perf_debugfs_run = { 727 .owner = THIS_MODULE, 728 .open = simple_open, 729 .read = debugfs_run_read, 730 .write = debugfs_run_write, 731 }; 732 733 static int perf_debugfs_setup(struct perf_ctx *perf) 734 { 735 struct pci_dev *pdev = perf->ntb->pdev; 736 struct dentry *debugfs_node_dir; 737 struct dentry *debugfs_run; 738 struct dentry *debugfs_threads; 739 struct dentry *debugfs_seg_order; 740 struct dentry *debugfs_run_order; 741 struct dentry *debugfs_use_dma; 742 struct dentry *debugfs_on_node; 743 744 if (!debugfs_initialized()) 745 return -ENODEV; 746 747 /* Assumpion: only one NTB device in the system */ 748 if (!perf_debugfs_dir) { 749 perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 750 if (!perf_debugfs_dir) 751 return -ENODEV; 752 } 753 754 debugfs_node_dir = debugfs_create_dir(pci_name(pdev), 755 perf_debugfs_dir); 756 if (!debugfs_node_dir) 757 goto err; 758 759 debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR, 760 debugfs_node_dir, perf, 761 &ntb_perf_debugfs_run); 762 if (!debugfs_run) 763 goto err; 764 765 debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR, 766 debugfs_node_dir, 767 &perf->perf_threads); 768 if (!debugfs_threads) 769 goto err; 770 771 debugfs_seg_order = debugfs_create_u32("seg_order", 0600, 772 debugfs_node_dir, 773 &seg_order); 774 if (!debugfs_seg_order) 775 goto err; 776 777 debugfs_run_order = debugfs_create_u32("run_order", 0600, 778 debugfs_node_dir, 779 &run_order); 780 if (!debugfs_run_order) 781 goto err; 782 783 debugfs_use_dma = debugfs_create_bool("use_dma", 0600, 784 debugfs_node_dir, 785 &use_dma); 786 if (!debugfs_use_dma) 787 goto err; 788 789 debugfs_on_node = debugfs_create_bool("on_node", 0600, 790 debugfs_node_dir, 791 &on_node); 792 if (!debugfs_on_node) 793 goto err; 794 795 return 0; 796 797 err: 798 debugfs_remove_recursive(perf_debugfs_dir); 799 perf_debugfs_dir = NULL; 800 return -ENODEV; 801 } 802 803 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb) 804 { 805 struct pci_dev *pdev = ntb->pdev; 806 struct perf_ctx *perf; 807 int node; 808 int rc = 0; 809 810 if (ntb_spad_count(ntb) < MAX_SPAD) { 811 dev_err(&ntb->dev, "Not enough scratch pad registers for %s", 812 DRIVER_NAME); 813 return -EIO; 814 } 815 816 if (!ntb->ops->mw_set_trans) { 817 dev_err(&ntb->dev, "Need inbound MW based NTB API\n"); 818 return -EINVAL; 819 } 820 821 if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT) 822 dev_warn(&ntb->dev, "Multi-port NTB devices unsupported\n"); 823 824 node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE; 825 perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node); 826 if (!perf) { 827 rc = -ENOMEM; 828 goto err_perf; 829 } 830 831 perf->ntb = ntb; 832 perf->perf_threads = 1; 833 atomic_set(&perf->tsync, 0); 834 mutex_init(&perf->run_mutex); 835 spin_lock_init(&perf->db_lock); 836 perf_setup_mw(ntb, perf); 837 init_waitqueue_head(&perf->link_wq); 838 INIT_DELAYED_WORK(&perf->link_work, perf_link_work); 839 840 rc = ntb_set_ctx(ntb, perf, &perf_ops); 841 if (rc) 842 goto err_ctx; 843 844 perf->link_is_up = false; 845 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 846 ntb_link_event(ntb); 847 848 rc = perf_debugfs_setup(perf); 849 if (rc) 850 goto err_ctx; 851 852 perf_clear_thread_status(perf); 853 854 return 0; 855 856 err_ctx: 857 cancel_delayed_work_sync(&perf->link_work); 858 kfree(perf); 859 err_perf: 860 return rc; 861 } 862 863 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb) 864 { 865 struct perf_ctx *perf = ntb->ctx; 866 int i; 867 868 dev_dbg(&perf->ntb->dev, "%s called\n", __func__); 869 870 mutex_lock(&perf->run_mutex); 871 872 cancel_delayed_work_sync(&perf->link_work); 873 874 ntb_clear_ctx(ntb); 875 ntb_link_disable(ntb); 876 877 debugfs_remove_recursive(perf_debugfs_dir); 878 perf_debugfs_dir = NULL; 879 880 if (use_dma) { 881 for (i = 0; i < MAX_THREADS; i++) { 882 struct pthr_ctx *pctx = &perf->pthr_ctx[i]; 883 884 if (pctx->dma_chan) 885 dma_release_channel(pctx->dma_chan); 886 } 887 } 888 889 kfree(perf); 890 } 891 892 static struct ntb_client perf_client = { 893 .ops = { 894 .probe = perf_probe, 895 .remove = perf_remove, 896 }, 897 }; 898 module_ntb_client(perf_client); 899