1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * RapidIO mport character device 4 * 5 * Copyright 2014-2015 Integrated Device Technology, Inc. 6 * Alexandre Bounine <alexandre.bounine@idt.com> 7 * Copyright 2014-2015 Prodrive Technologies 8 * Andre van Herk <andre.van.herk@prodrive-technologies.com> 9 * Jerry Jacobs <jerry.jacobs@prodrive-technologies.com> 10 * Copyright (C) 2014 Texas Instruments Incorporated 11 * Aurelien Jacquiot <a-jacquiot@ti.com> 12 */ 13 #include <linux/module.h> 14 #include <linux/kernel.h> 15 #include <linux/cdev.h> 16 #include <linux/ioctl.h> 17 #include <linux/uaccess.h> 18 #include <linux/list.h> 19 #include <linux/fs.h> 20 #include <linux/err.h> 21 #include <linux/net.h> 22 #include <linux/poll.h> 23 #include <linux/spinlock.h> 24 #include <linux/sched.h> 25 #include <linux/kfifo.h> 26 27 #include <linux/mm.h> 28 #include <linux/slab.h> 29 #include <linux/vmalloc.h> 30 #include <linux/mman.h> 31 32 #include <linux/dma-mapping.h> 33 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 34 #include <linux/dmaengine.h> 35 #endif 36 37 #include <linux/rio.h> 38 #include <linux/rio_ids.h> 39 #include <linux/rio_drv.h> 40 #include <linux/rio_mport_cdev.h> 41 42 #include "../rio.h" 43 44 #define DRV_NAME "rio_mport" 45 #define DRV_PREFIX DRV_NAME ": " 46 #define DEV_NAME "rio_mport" 47 #define DRV_VERSION "1.0.0" 48 49 /* Debug output filtering masks */ 50 enum { 51 DBG_NONE = 0, 52 DBG_INIT = BIT(0), /* driver init */ 53 DBG_EXIT = BIT(1), /* driver exit */ 54 DBG_MPORT = BIT(2), /* mport add/remove */ 55 DBG_RDEV = BIT(3), /* RapidIO device add/remove */ 56 DBG_DMA = BIT(4), /* DMA transfer messages */ 57 DBG_MMAP = BIT(5), /* mapping messages */ 58 DBG_IBW = BIT(6), /* inbound window */ 59 DBG_EVENT = BIT(7), /* event handling messages */ 60 DBG_OBW = BIT(8), /* outbound window messages */ 61 DBG_DBELL = BIT(9), /* doorbell messages */ 62 DBG_ALL = ~0, 63 }; 64 65 #ifdef DEBUG 66 #define rmcd_debug(level, fmt, arg...) \ 67 do { \ 68 if (DBG_##level & dbg_level) \ 69 pr_debug(DRV_PREFIX "%s: " fmt "\n", __func__, ##arg); \ 70 } while (0) 71 #else 72 #define rmcd_debug(level, fmt, arg...) \ 73 no_printk(KERN_DEBUG pr_fmt(DRV_PREFIX fmt "\n"), ##arg) 74 #endif 75 76 #define rmcd_warn(fmt, arg...) \ 77 pr_warn(DRV_PREFIX "%s WARNING " fmt "\n", __func__, ##arg) 78 79 #define rmcd_error(fmt, arg...) \ 80 pr_err(DRV_PREFIX "%s ERROR " fmt "\n", __func__, ##arg) 81 82 MODULE_AUTHOR("Jerry Jacobs <jerry.jacobs@prodrive-technologies.com>"); 83 MODULE_AUTHOR("Aurelien Jacquiot <a-jacquiot@ti.com>"); 84 MODULE_AUTHOR("Alexandre Bounine <alexandre.bounine@idt.com>"); 85 MODULE_AUTHOR("Andre van Herk <andre.van.herk@prodrive-technologies.com>"); 86 MODULE_DESCRIPTION("RapidIO mport character device driver"); 87 MODULE_LICENSE("GPL"); 88 MODULE_VERSION(DRV_VERSION); 89 90 static int dma_timeout = 3000; /* DMA transfer timeout in msec */ 91 module_param(dma_timeout, int, S_IRUGO); 92 MODULE_PARM_DESC(dma_timeout, "DMA Transfer Timeout in msec (default: 3000)"); 93 94 #ifdef DEBUG 95 static u32 dbg_level = DBG_NONE; 96 module_param(dbg_level, uint, S_IWUSR | S_IWGRP | S_IRUGO); 97 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)"); 98 #endif 99 100 /* 101 * Internal memory mapping structure 102 */ 103 enum rio_mport_map_dir { 104 MAP_INBOUND, 105 MAP_OUTBOUND, 106 MAP_DMA, 107 }; 108 109 struct rio_mport_mapping { 110 struct list_head node; 111 struct mport_dev *md; 112 enum rio_mport_map_dir dir; 113 u16 rioid; 114 u64 rio_addr; 115 dma_addr_t phys_addr; /* for mmap */ 116 void *virt_addr; /* kernel address, for dma_free_coherent */ 117 u64 size; 118 struct kref ref; /* refcount of vmas sharing the mapping */ 119 struct file *filp; 120 }; 121 122 #define MPORT_EVENT_DEPTH 10 123 124 /* 125 * mport_dev driver-specific structure that represents mport device 126 * @active mport device status flag 127 * @node list node to maintain list of registered mports 128 * @cdev character device 129 * @dev associated device object 130 * @mport associated subsystem's master port device object 131 * @buf_mutex lock for buffer handling 132 * @file_mutex - lock for open files list 133 * @file_list - list of open files on given mport 134 * @properties properties of this mport 135 * @portwrites queue of inbound portwrites 136 * @pw_lock lock for port write queue 137 * @mappings queue for memory mappings 138 * @dma_chan DMA channels associated with this device 139 * @dma_ref: 140 * @comp: 141 */ 142 struct mport_dev { 143 atomic_t active; 144 struct list_head node; 145 struct cdev cdev; 146 struct device dev; 147 struct rio_mport *mport; 148 struct mutex buf_mutex; 149 struct mutex file_mutex; 150 struct list_head file_list; 151 struct rio_mport_properties properties; 152 struct list_head doorbells; 153 spinlock_t db_lock; 154 struct list_head portwrites; 155 spinlock_t pw_lock; 156 struct list_head mappings; 157 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 158 struct dma_chan *dma_chan; 159 struct kref dma_ref; 160 struct completion comp; 161 #endif 162 }; 163 164 /* 165 * mport_cdev_priv - data structure specific to individual file object 166 * associated with an open device 167 * @md master port character device object 168 * @async_queue - asynchronous notification queue 169 * @list - file objects tracking list 170 * @db_filters inbound doorbell filters for this descriptor 171 * @pw_filters portwrite filters for this descriptor 172 * @event_fifo event fifo for this descriptor 173 * @event_rx_wait wait queue for this descriptor 174 * @fifo_lock lock for event_fifo 175 * @event_mask event mask for this descriptor 176 * @dmach DMA engine channel allocated for specific file object 177 */ 178 struct mport_cdev_priv { 179 struct mport_dev *md; 180 struct fasync_struct *async_queue; 181 struct list_head list; 182 struct list_head db_filters; 183 struct list_head pw_filters; 184 struct kfifo event_fifo; 185 wait_queue_head_t event_rx_wait; 186 spinlock_t fifo_lock; 187 u32 event_mask; /* RIO_DOORBELL, RIO_PORTWRITE */ 188 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 189 struct dma_chan *dmach; 190 struct list_head async_list; 191 spinlock_t req_lock; 192 struct mutex dma_lock; 193 struct kref dma_ref; 194 struct completion comp; 195 #endif 196 }; 197 198 /* 199 * rio_mport_pw_filter - structure to describe a portwrite filter 200 * md_node node in mport device's list 201 * priv_node node in private file object's list 202 * priv reference to private data 203 * filter actual portwrite filter 204 */ 205 struct rio_mport_pw_filter { 206 struct list_head md_node; 207 struct list_head priv_node; 208 struct mport_cdev_priv *priv; 209 struct rio_pw_filter filter; 210 }; 211 212 /* 213 * rio_mport_db_filter - structure to describe a doorbell filter 214 * @data_node reference to device node 215 * @priv_node node in private data 216 * @priv reference to private data 217 * @filter actual doorbell filter 218 */ 219 struct rio_mport_db_filter { 220 struct list_head data_node; 221 struct list_head priv_node; 222 struct mport_cdev_priv *priv; 223 struct rio_doorbell_filter filter; 224 }; 225 226 static LIST_HEAD(mport_devs); 227 static DEFINE_MUTEX(mport_devs_lock); 228 229 #if (0) /* used by commented out portion of poll function : FIXME */ 230 static DECLARE_WAIT_QUEUE_HEAD(mport_cdev_wait); 231 #endif 232 233 static const struct class dev_class = { 234 .name = DRV_NAME, 235 }; 236 static dev_t dev_number; 237 238 static void mport_release_mapping(struct kref *ref); 239 240 static int rio_mport_maint_rd(struct mport_cdev_priv *priv, void __user *arg, 241 int local) 242 { 243 struct rio_mport *mport = priv->md->mport; 244 struct rio_mport_maint_io maint_io; 245 u32 *buffer; 246 u32 offset; 247 size_t length; 248 int ret, i; 249 250 if (unlikely(copy_from_user(&maint_io, arg, sizeof(maint_io)))) 251 return -EFAULT; 252 253 if ((maint_io.offset % 4) || 254 (maint_io.length == 0) || (maint_io.length % 4) || 255 (maint_io.length + maint_io.offset) > RIO_MAINT_SPACE_SZ) 256 return -EINVAL; 257 258 buffer = vmalloc(maint_io.length); 259 if (buffer == NULL) 260 return -ENOMEM; 261 length = maint_io.length/sizeof(u32); 262 offset = maint_io.offset; 263 264 for (i = 0; i < length; i++) { 265 if (local) 266 ret = __rio_local_read_config_32(mport, 267 offset, &buffer[i]); 268 else 269 ret = rio_mport_read_config_32(mport, maint_io.rioid, 270 maint_io.hopcount, offset, &buffer[i]); 271 if (ret) 272 goto out; 273 274 offset += 4; 275 } 276 277 if (unlikely(copy_to_user((void __user *)(uintptr_t)maint_io.buffer, 278 buffer, maint_io.length))) 279 ret = -EFAULT; 280 out: 281 vfree(buffer); 282 return ret; 283 } 284 285 static int rio_mport_maint_wr(struct mport_cdev_priv *priv, void __user *arg, 286 int local) 287 { 288 struct rio_mport *mport = priv->md->mport; 289 struct rio_mport_maint_io maint_io; 290 u32 *buffer; 291 u32 offset; 292 size_t length; 293 int ret = -EINVAL, i; 294 295 if (unlikely(copy_from_user(&maint_io, arg, sizeof(maint_io)))) 296 return -EFAULT; 297 298 if ((maint_io.offset % 4) || 299 (maint_io.length == 0) || (maint_io.length % 4) || 300 (maint_io.length + maint_io.offset) > RIO_MAINT_SPACE_SZ) 301 return -EINVAL; 302 303 buffer = vmalloc(maint_io.length); 304 if (buffer == NULL) 305 return -ENOMEM; 306 length = maint_io.length; 307 308 if (unlikely(copy_from_user(buffer, 309 (void __user *)(uintptr_t)maint_io.buffer, length))) { 310 ret = -EFAULT; 311 goto out; 312 } 313 314 offset = maint_io.offset; 315 length /= sizeof(u32); 316 317 for (i = 0; i < length; i++) { 318 if (local) 319 ret = __rio_local_write_config_32(mport, 320 offset, buffer[i]); 321 else 322 ret = rio_mport_write_config_32(mport, maint_io.rioid, 323 maint_io.hopcount, 324 offset, buffer[i]); 325 if (ret) 326 goto out; 327 328 offset += 4; 329 } 330 331 out: 332 vfree(buffer); 333 return ret; 334 } 335 336 337 /* 338 * Inbound/outbound memory mapping functions 339 */ 340 static int 341 rio_mport_create_outbound_mapping(struct mport_dev *md, struct file *filp, 342 u16 rioid, u64 raddr, u32 size, 343 dma_addr_t *paddr) 344 { 345 struct rio_mport *mport = md->mport; 346 struct rio_mport_mapping *map; 347 int ret; 348 349 rmcd_debug(OBW, "did=%d ra=0x%llx sz=0x%x", rioid, raddr, size); 350 351 map = kzalloc_obj(*map); 352 if (map == NULL) 353 return -ENOMEM; 354 355 ret = rio_map_outb_region(mport, rioid, raddr, size, 0, paddr); 356 if (ret < 0) 357 goto err_map_outb; 358 359 map->dir = MAP_OUTBOUND; 360 map->rioid = rioid; 361 map->rio_addr = raddr; 362 map->size = size; 363 map->phys_addr = *paddr; 364 map->filp = filp; 365 map->md = md; 366 kref_init(&map->ref); 367 list_add_tail(&map->node, &md->mappings); 368 return 0; 369 err_map_outb: 370 kfree(map); 371 return ret; 372 } 373 374 static int 375 rio_mport_get_outbound_mapping(struct mport_dev *md, struct file *filp, 376 u16 rioid, u64 raddr, u32 size, 377 dma_addr_t *paddr) 378 { 379 struct rio_mport_mapping *map; 380 int err = -ENOMEM; 381 382 mutex_lock(&md->buf_mutex); 383 list_for_each_entry(map, &md->mappings, node) { 384 if (map->dir != MAP_OUTBOUND) 385 continue; 386 if (rioid == map->rioid && 387 raddr == map->rio_addr && size == map->size) { 388 *paddr = map->phys_addr; 389 err = 0; 390 break; 391 } else if (rioid == map->rioid && 392 raddr < (map->rio_addr + map->size - 1) && 393 (raddr + size) > map->rio_addr) { 394 err = -EBUSY; 395 break; 396 } 397 } 398 399 /* If not found, create new */ 400 if (err == -ENOMEM) 401 err = rio_mport_create_outbound_mapping(md, filp, rioid, raddr, 402 size, paddr); 403 mutex_unlock(&md->buf_mutex); 404 return err; 405 } 406 407 static int rio_mport_obw_map(struct file *filp, void __user *arg) 408 { 409 struct mport_cdev_priv *priv = filp->private_data; 410 struct mport_dev *data = priv->md; 411 struct rio_mmap map; 412 dma_addr_t paddr; 413 int ret; 414 415 if (unlikely(copy_from_user(&map, arg, sizeof(map)))) 416 return -EFAULT; 417 418 rmcd_debug(OBW, "did=%d ra=0x%llx sz=0x%llx", 419 map.rioid, map.rio_addr, map.length); 420 421 ret = rio_mport_get_outbound_mapping(data, filp, map.rioid, 422 map.rio_addr, map.length, &paddr); 423 if (ret < 0) { 424 rmcd_error("Failed to set OBW err= %d", ret); 425 return ret; 426 } 427 428 map.handle = paddr; 429 430 if (unlikely(copy_to_user(arg, &map, sizeof(map)))) 431 return -EFAULT; 432 return 0; 433 } 434 435 /* 436 * rio_mport_obw_free() - unmap an OutBound Window from RapidIO address space 437 * 438 * @priv: driver private data 439 * @arg: buffer handle returned by allocation routine 440 */ 441 static int rio_mport_obw_free(struct file *filp, void __user *arg) 442 { 443 struct mport_cdev_priv *priv = filp->private_data; 444 struct mport_dev *md = priv->md; 445 u64 handle; 446 struct rio_mport_mapping *map, *_map; 447 448 if (!md->mport->ops->unmap_outb) 449 return -EPROTONOSUPPORT; 450 451 if (copy_from_user(&handle, arg, sizeof(handle))) 452 return -EFAULT; 453 454 rmcd_debug(OBW, "h=0x%llx", handle); 455 456 mutex_lock(&md->buf_mutex); 457 list_for_each_entry_safe(map, _map, &md->mappings, node) { 458 if (map->dir == MAP_OUTBOUND && map->phys_addr == handle) { 459 if (map->filp == filp) { 460 rmcd_debug(OBW, "kref_put h=0x%llx", handle); 461 map->filp = NULL; 462 kref_put(&map->ref, mport_release_mapping); 463 } 464 break; 465 } 466 } 467 mutex_unlock(&md->buf_mutex); 468 469 return 0; 470 } 471 472 /* 473 * maint_hdid_set() - Set the host Device ID 474 * @priv: driver private data 475 * @arg: Device Id 476 */ 477 static int maint_hdid_set(struct mport_cdev_priv *priv, void __user *arg) 478 { 479 struct mport_dev *md = priv->md; 480 u16 hdid; 481 482 if (copy_from_user(&hdid, arg, sizeof(hdid))) 483 return -EFAULT; 484 485 md->mport->host_deviceid = hdid; 486 md->properties.hdid = hdid; 487 rio_local_set_device_id(md->mport, hdid); 488 489 rmcd_debug(MPORT, "Set host device Id to %d", hdid); 490 491 return 0; 492 } 493 494 /* 495 * maint_comptag_set() - Set the host Component Tag 496 * @priv: driver private data 497 * @arg: Component Tag 498 */ 499 static int maint_comptag_set(struct mport_cdev_priv *priv, void __user *arg) 500 { 501 struct mport_dev *md = priv->md; 502 u32 comptag; 503 504 if (copy_from_user(&comptag, arg, sizeof(comptag))) 505 return -EFAULT; 506 507 rio_local_write_config_32(md->mport, RIO_COMPONENT_TAG_CSR, comptag); 508 509 rmcd_debug(MPORT, "Set host Component Tag to %d", comptag); 510 511 return 0; 512 } 513 514 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 515 516 struct mport_dma_req { 517 struct kref refcount; 518 struct list_head node; 519 struct file *filp; 520 struct mport_cdev_priv *priv; 521 enum rio_transfer_sync sync; 522 struct sg_table sgt; 523 struct page **page_list; 524 unsigned int nr_pages; 525 struct rio_mport_mapping *map; 526 struct dma_chan *dmach; 527 enum dma_data_direction dir; 528 dma_cookie_t cookie; 529 enum dma_status status; 530 struct completion req_comp; 531 }; 532 533 static void mport_release_def_dma(struct kref *dma_ref) 534 { 535 struct mport_dev *md = 536 container_of(dma_ref, struct mport_dev, dma_ref); 537 538 rmcd_debug(EXIT, "DMA_%d", md->dma_chan->chan_id); 539 rio_release_dma(md->dma_chan); 540 md->dma_chan = NULL; 541 } 542 543 static void mport_release_dma(struct kref *dma_ref) 544 { 545 struct mport_cdev_priv *priv = 546 container_of(dma_ref, struct mport_cdev_priv, dma_ref); 547 548 rmcd_debug(EXIT, "DMA_%d", priv->dmach->chan_id); 549 complete(&priv->comp); 550 } 551 552 static void dma_req_free(struct kref *ref) 553 { 554 struct mport_dma_req *req = container_of(ref, struct mport_dma_req, 555 refcount); 556 struct mport_cdev_priv *priv = req->priv; 557 558 dma_unmap_sg(req->dmach->device->dev, 559 req->sgt.sgl, req->sgt.nents, req->dir); 560 sg_free_table(&req->sgt); 561 if (req->page_list) { 562 unpin_user_pages(req->page_list, req->nr_pages); 563 kfree(req->page_list); 564 } 565 566 if (req->map) { 567 struct rio_mport_mapping *map = req->map; 568 struct mport_dev *md = map->md; 569 570 mutex_lock(&md->buf_mutex); 571 req->map = NULL; 572 kref_put(&map->ref, mport_release_mapping); 573 mutex_unlock(&md->buf_mutex); 574 } 575 576 kref_put(&priv->dma_ref, mport_release_dma); 577 578 kfree(req); 579 } 580 581 static void dma_xfer_callback(void *param) 582 { 583 struct mport_dma_req *req = (struct mport_dma_req *)param; 584 struct mport_cdev_priv *priv = req->priv; 585 586 req->status = dma_async_is_tx_complete(priv->dmach, req->cookie, 587 NULL, NULL); 588 complete(&req->req_comp); 589 kref_put(&req->refcount, dma_req_free); 590 } 591 592 /* 593 * prep_dma_xfer() - Configure and send request to DMAengine to prepare DMA 594 * transfer object. 595 * Returns pointer to DMA transaction descriptor allocated by DMA driver on 596 * success or ERR_PTR (and/or NULL) if failed. Caller must check returned 597 * non-NULL pointer using IS_ERR macro. 598 */ 599 static struct dma_async_tx_descriptor 600 *prep_dma_xfer(struct dma_chan *chan, struct rio_transfer_io *transfer, 601 struct sg_table *sgt, int nents, enum dma_transfer_direction dir, 602 enum dma_ctrl_flags flags) 603 { 604 struct rio_dma_data tx_data; 605 606 tx_data.sg = sgt->sgl; 607 tx_data.sg_len = nents; 608 tx_data.rio_addr_u = 0; 609 tx_data.rio_addr = transfer->rio_addr; 610 if (dir == DMA_MEM_TO_DEV) { 611 switch (transfer->method) { 612 case RIO_EXCHANGE_NWRITE: 613 tx_data.wr_type = RDW_ALL_NWRITE; 614 break; 615 case RIO_EXCHANGE_NWRITE_R_ALL: 616 tx_data.wr_type = RDW_ALL_NWRITE_R; 617 break; 618 case RIO_EXCHANGE_NWRITE_R: 619 tx_data.wr_type = RDW_LAST_NWRITE_R; 620 break; 621 case RIO_EXCHANGE_DEFAULT: 622 tx_data.wr_type = RDW_DEFAULT; 623 break; 624 default: 625 return ERR_PTR(-EINVAL); 626 } 627 } 628 629 return rio_dma_prep_xfer(chan, transfer->rioid, &tx_data, dir, flags); 630 } 631 632 /* Request DMA channel associated with this mport device. 633 * Try to request DMA channel for every new process that opened given 634 * mport. If a new DMA channel is not available use default channel 635 * which is the first DMA channel opened on mport device. 636 */ 637 static int get_dma_channel(struct mport_cdev_priv *priv) 638 { 639 mutex_lock(&priv->dma_lock); 640 if (!priv->dmach) { 641 priv->dmach = rio_request_mport_dma(priv->md->mport); 642 if (!priv->dmach) { 643 /* Use default DMA channel if available */ 644 if (priv->md->dma_chan) { 645 priv->dmach = priv->md->dma_chan; 646 kref_get(&priv->md->dma_ref); 647 } else { 648 rmcd_error("Failed to get DMA channel"); 649 mutex_unlock(&priv->dma_lock); 650 return -ENODEV; 651 } 652 } else if (!priv->md->dma_chan) { 653 /* Register default DMA channel if we do not have one */ 654 priv->md->dma_chan = priv->dmach; 655 kref_init(&priv->md->dma_ref); 656 rmcd_debug(DMA, "Register DMA_chan %d as default", 657 priv->dmach->chan_id); 658 } 659 660 kref_init(&priv->dma_ref); 661 init_completion(&priv->comp); 662 } 663 664 kref_get(&priv->dma_ref); 665 mutex_unlock(&priv->dma_lock); 666 return 0; 667 } 668 669 static void put_dma_channel(struct mport_cdev_priv *priv) 670 { 671 kref_put(&priv->dma_ref, mport_release_dma); 672 } 673 674 /* 675 * DMA transfer functions 676 */ 677 static int do_dma_request(struct mport_dma_req *req, 678 struct rio_transfer_io *xfer, 679 enum rio_transfer_sync sync, int nents) 680 { 681 struct mport_cdev_priv *priv; 682 struct sg_table *sgt; 683 struct dma_chan *chan; 684 struct dma_async_tx_descriptor *tx; 685 dma_cookie_t cookie; 686 unsigned long tmo = msecs_to_jiffies(dma_timeout); 687 enum dma_transfer_direction dir; 688 long wret; 689 int ret = 0; 690 691 priv = req->priv; 692 sgt = &req->sgt; 693 694 chan = priv->dmach; 695 dir = (req->dir == DMA_FROM_DEVICE) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; 696 697 rmcd_debug(DMA, "%s(%d) uses %s for DMA_%s", 698 current->comm, task_pid_nr(current), 699 dev_name(&chan->dev->device), 700 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE"); 701 702 /* Initialize DMA transaction request */ 703 tx = prep_dma_xfer(chan, xfer, sgt, nents, dir, 704 DMA_CTRL_ACK | DMA_PREP_INTERRUPT); 705 706 if (!tx) { 707 rmcd_debug(DMA, "prep error for %s A:0x%llx L:0x%llx", 708 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", 709 xfer->rio_addr, xfer->length); 710 ret = -EIO; 711 goto err_out; 712 } else if (IS_ERR(tx)) { 713 ret = PTR_ERR(tx); 714 rmcd_debug(DMA, "prep error %d for %s A:0x%llx L:0x%llx", ret, 715 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", 716 xfer->rio_addr, xfer->length); 717 goto err_out; 718 } 719 720 tx->callback = dma_xfer_callback; 721 tx->callback_param = req; 722 723 req->status = DMA_IN_PROGRESS; 724 kref_get(&req->refcount); 725 726 cookie = dmaengine_submit(tx); 727 req->cookie = cookie; 728 729 rmcd_debug(DMA, "pid=%d DMA_%s tx_cookie = %d", task_pid_nr(current), 730 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie); 731 732 if (dma_submit_error(cookie)) { 733 rmcd_error("submit err=%d (addr:0x%llx len:0x%llx)", 734 cookie, xfer->rio_addr, xfer->length); 735 kref_put(&req->refcount, dma_req_free); 736 ret = -EIO; 737 goto err_out; 738 } 739 740 dma_async_issue_pending(chan); 741 742 if (sync == RIO_TRANSFER_ASYNC) { 743 spin_lock(&priv->req_lock); 744 list_add_tail(&req->node, &priv->async_list); 745 spin_unlock(&priv->req_lock); 746 return cookie; 747 } else if (sync == RIO_TRANSFER_FAF) 748 return 0; 749 750 wret = wait_for_completion_interruptible_timeout(&req->req_comp, tmo); 751 752 if (wret == 0) { 753 /* Timeout on wait occurred */ 754 rmcd_error("%s(%d) timed out waiting for DMA_%s %d", 755 current->comm, task_pid_nr(current), 756 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie); 757 return -ETIMEDOUT; 758 } else if (wret == -ERESTARTSYS) { 759 /* Wait_for_completion was interrupted by a signal but DMA may 760 * be in progress 761 */ 762 rmcd_error("%s(%d) wait for DMA_%s %d was interrupted", 763 current->comm, task_pid_nr(current), 764 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", cookie); 765 return -EINTR; 766 } 767 768 if (req->status != DMA_COMPLETE) { 769 /* DMA transaction completion was signaled with error */ 770 rmcd_error("%s(%d) DMA_%s %d completed with status %d (ret=%d)", 771 current->comm, task_pid_nr(current), 772 (dir == DMA_DEV_TO_MEM)?"READ":"WRITE", 773 cookie, req->status, ret); 774 ret = -EIO; 775 } 776 777 err_out: 778 return ret; 779 } 780 781 /* 782 * rio_dma_transfer() - Perform RapidIO DMA data transfer to/from 783 * the remote RapidIO device 784 * @filp: file pointer associated with the call 785 * @transfer_mode: DMA transfer mode 786 * @sync: synchronization mode 787 * @dir: DMA transfer direction (DMA_MEM_TO_DEV = write OR 788 * DMA_DEV_TO_MEM = read) 789 * @xfer: data transfer descriptor structure 790 */ 791 static int 792 rio_dma_transfer(struct file *filp, u32 transfer_mode, 793 enum rio_transfer_sync sync, enum dma_data_direction dir, 794 struct rio_transfer_io *xfer) 795 { 796 struct mport_cdev_priv *priv = filp->private_data; 797 unsigned long nr_pages = 0; 798 struct page **page_list = NULL; 799 struct mport_dma_req *req; 800 struct mport_dev *md = priv->md; 801 struct dma_chan *chan; 802 int ret; 803 int nents; 804 805 if (xfer->length == 0) 806 return -EINVAL; 807 req = kzalloc_obj(*req); 808 if (!req) 809 return -ENOMEM; 810 811 ret = get_dma_channel(priv); 812 if (ret) { 813 kfree(req); 814 return ret; 815 } 816 chan = priv->dmach; 817 818 kref_init(&req->refcount); 819 init_completion(&req->req_comp); 820 req->dir = dir; 821 req->filp = filp; 822 req->priv = priv; 823 req->dmach = chan; 824 req->sync = sync; 825 826 /* 827 * If parameter loc_addr != NULL, we are transferring data from/to 828 * data buffer allocated in user-space: lock in memory user-space 829 * buffer pages and build an SG table for DMA transfer request 830 * 831 * Otherwise (loc_addr == NULL) contiguous kernel-space buffer is 832 * used for DMA data transfers: build single entry SG table using 833 * offset within the internal buffer specified by handle parameter. 834 */ 835 if (xfer->loc_addr) { 836 unsigned int offset; 837 long pinned; 838 839 offset = lower_32_bits(offset_in_page(xfer->loc_addr)); 840 nr_pages = PAGE_ALIGN(xfer->length + offset) >> PAGE_SHIFT; 841 842 page_list = kmalloc_objs(*page_list, nr_pages); 843 if (page_list == NULL) { 844 ret = -ENOMEM; 845 goto err_req; 846 } 847 848 pinned = pin_user_pages_fast( 849 (unsigned long)xfer->loc_addr & PAGE_MASK, 850 nr_pages, 851 dir == DMA_FROM_DEVICE ? FOLL_WRITE : 0, 852 page_list); 853 854 if (pinned != nr_pages) { 855 if (pinned < 0) { 856 rmcd_error("pin_user_pages_fast err=%ld", 857 pinned); 858 nr_pages = 0; 859 } else { 860 rmcd_error("pinned %ld out of %ld pages", 861 pinned, nr_pages); 862 /* 863 * Set nr_pages up to mean "how many pages to unpin, in 864 * the error handler: 865 */ 866 nr_pages = pinned; 867 } 868 ret = -EFAULT; 869 goto err_pg; 870 } 871 872 ret = sg_alloc_table_from_pages(&req->sgt, page_list, nr_pages, 873 offset, xfer->length, GFP_KERNEL); 874 if (ret) { 875 rmcd_error("sg_alloc_table failed with err=%d", ret); 876 goto err_pg; 877 } 878 879 req->page_list = page_list; 880 req->nr_pages = nr_pages; 881 } else { 882 dma_addr_t baddr; 883 struct rio_mport_mapping *map; 884 885 baddr = (dma_addr_t)xfer->handle; 886 887 mutex_lock(&md->buf_mutex); 888 list_for_each_entry(map, &md->mappings, node) { 889 if (baddr >= map->phys_addr && 890 baddr < (map->phys_addr + map->size)) { 891 kref_get(&map->ref); 892 req->map = map; 893 break; 894 } 895 } 896 mutex_unlock(&md->buf_mutex); 897 898 if (req->map == NULL) { 899 ret = -ENOMEM; 900 goto err_req; 901 } 902 903 if (xfer->length + xfer->offset > req->map->size) { 904 ret = -EINVAL; 905 goto err_req; 906 } 907 908 ret = sg_alloc_table(&req->sgt, 1, GFP_KERNEL); 909 if (unlikely(ret)) { 910 rmcd_error("sg_alloc_table failed for internal buf"); 911 goto err_req; 912 } 913 914 sg_set_buf(req->sgt.sgl, 915 req->map->virt_addr + (baddr - req->map->phys_addr) + 916 xfer->offset, xfer->length); 917 } 918 919 nents = dma_map_sg(chan->device->dev, 920 req->sgt.sgl, req->sgt.nents, dir); 921 if (nents == 0) { 922 rmcd_error("Failed to map SG list"); 923 ret = -EFAULT; 924 goto err_pg; 925 } 926 927 ret = do_dma_request(req, xfer, sync, nents); 928 929 if (ret >= 0) { 930 if (sync == RIO_TRANSFER_ASYNC) 931 return ret; /* return ASYNC cookie */ 932 } else { 933 rmcd_debug(DMA, "do_dma_request failed with err=%d", ret); 934 } 935 936 err_pg: 937 if (!req->page_list) { 938 unpin_user_pages(page_list, nr_pages); 939 kfree(page_list); 940 } 941 err_req: 942 kref_put(&req->refcount, dma_req_free); 943 return ret; 944 } 945 946 static int rio_mport_transfer_ioctl(struct file *filp, void __user *arg) 947 { 948 struct mport_cdev_priv *priv = filp->private_data; 949 struct rio_transaction transaction; 950 struct rio_transfer_io *transfer; 951 enum dma_data_direction dir; 952 int i, ret = 0; 953 size_t size; 954 955 if (unlikely(copy_from_user(&transaction, arg, sizeof(transaction)))) 956 return -EFAULT; 957 958 if (transaction.count != 1) /* only single transfer for now */ 959 return -EINVAL; 960 961 if ((transaction.transfer_mode & 962 priv->md->properties.transfer_mode) == 0) 963 return -ENODEV; 964 965 size = array_size(sizeof(*transfer), transaction.count); 966 transfer = vmalloc(size); 967 if (!transfer) 968 return -ENOMEM; 969 970 if (unlikely(copy_from_user(transfer, 971 (void __user *)(uintptr_t)transaction.block, 972 size))) { 973 ret = -EFAULT; 974 goto out_free; 975 } 976 977 dir = (transaction.dir == RIO_TRANSFER_DIR_READ) ? 978 DMA_FROM_DEVICE : DMA_TO_DEVICE; 979 for (i = 0; i < transaction.count && ret == 0; i++) 980 ret = rio_dma_transfer(filp, transaction.transfer_mode, 981 transaction.sync, dir, &transfer[i]); 982 983 if (unlikely(copy_to_user((void __user *)(uintptr_t)transaction.block, 984 transfer, size))) 985 ret = -EFAULT; 986 987 out_free: 988 vfree(transfer); 989 990 return ret; 991 } 992 993 static int rio_mport_wait_for_async_dma(struct file *filp, void __user *arg) 994 { 995 struct mport_cdev_priv *priv; 996 struct rio_async_tx_wait w_param; 997 struct mport_dma_req *req; 998 dma_cookie_t cookie; 999 unsigned long tmo; 1000 long wret; 1001 int found = 0; 1002 int ret; 1003 1004 priv = (struct mport_cdev_priv *)filp->private_data; 1005 1006 if (unlikely(copy_from_user(&w_param, arg, sizeof(w_param)))) 1007 return -EFAULT; 1008 1009 cookie = w_param.token; 1010 if (w_param.timeout) 1011 tmo = msecs_to_jiffies(w_param.timeout); 1012 else /* Use default DMA timeout */ 1013 tmo = msecs_to_jiffies(dma_timeout); 1014 1015 spin_lock(&priv->req_lock); 1016 list_for_each_entry(req, &priv->async_list, node) { 1017 if (req->cookie == cookie) { 1018 list_del(&req->node); 1019 found = 1; 1020 break; 1021 } 1022 } 1023 spin_unlock(&priv->req_lock); 1024 1025 if (!found) 1026 return -EAGAIN; 1027 1028 wret = wait_for_completion_interruptible_timeout(&req->req_comp, tmo); 1029 1030 if (wret == 0) { 1031 /* Timeout on wait occurred */ 1032 rmcd_error("%s(%d) timed out waiting for ASYNC DMA_%s", 1033 current->comm, task_pid_nr(current), 1034 (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE"); 1035 ret = -ETIMEDOUT; 1036 goto err_tmo; 1037 } else if (wret == -ERESTARTSYS) { 1038 /* Wait_for_completion was interrupted by a signal but DMA may 1039 * be still in progress 1040 */ 1041 rmcd_error("%s(%d) wait for ASYNC DMA_%s was interrupted", 1042 current->comm, task_pid_nr(current), 1043 (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE"); 1044 ret = -EINTR; 1045 goto err_tmo; 1046 } 1047 1048 if (req->status != DMA_COMPLETE) { 1049 /* DMA transaction completion signaled with transfer error */ 1050 rmcd_error("%s(%d) ASYNC DMA_%s completion with status %d", 1051 current->comm, task_pid_nr(current), 1052 (req->dir == DMA_FROM_DEVICE)?"READ":"WRITE", 1053 req->status); 1054 ret = -EIO; 1055 } else 1056 ret = 0; 1057 1058 if (req->status != DMA_IN_PROGRESS && req->status != DMA_PAUSED) 1059 kref_put(&req->refcount, dma_req_free); 1060 1061 return ret; 1062 1063 err_tmo: 1064 /* Return request back into async queue */ 1065 spin_lock(&priv->req_lock); 1066 list_add_tail(&req->node, &priv->async_list); 1067 spin_unlock(&priv->req_lock); 1068 return ret; 1069 } 1070 1071 static int rio_mport_create_dma_mapping(struct mport_dev *md, struct file *filp, 1072 u64 size, struct rio_mport_mapping **mapping) 1073 { 1074 struct rio_mport_mapping *map; 1075 1076 map = kzalloc_obj(*map); 1077 if (map == NULL) 1078 return -ENOMEM; 1079 1080 map->virt_addr = dma_alloc_coherent(md->mport->dev.parent, size, 1081 &map->phys_addr, GFP_KERNEL); 1082 if (map->virt_addr == NULL) { 1083 kfree(map); 1084 return -ENOMEM; 1085 } 1086 1087 map->dir = MAP_DMA; 1088 map->size = size; 1089 map->filp = filp; 1090 map->md = md; 1091 kref_init(&map->ref); 1092 mutex_lock(&md->buf_mutex); 1093 list_add_tail(&map->node, &md->mappings); 1094 mutex_unlock(&md->buf_mutex); 1095 *mapping = map; 1096 1097 return 0; 1098 } 1099 1100 static int rio_mport_alloc_dma(struct file *filp, void __user *arg) 1101 { 1102 struct mport_cdev_priv *priv = filp->private_data; 1103 struct mport_dev *md = priv->md; 1104 struct rio_dma_mem map; 1105 struct rio_mport_mapping *mapping = NULL; 1106 int ret; 1107 1108 if (unlikely(copy_from_user(&map, arg, sizeof(map)))) 1109 return -EFAULT; 1110 1111 ret = rio_mport_create_dma_mapping(md, filp, map.length, &mapping); 1112 if (ret) 1113 return ret; 1114 1115 map.dma_handle = mapping->phys_addr; 1116 1117 if (unlikely(copy_to_user(arg, &map, sizeof(map)))) { 1118 mutex_lock(&md->buf_mutex); 1119 kref_put(&mapping->ref, mport_release_mapping); 1120 mutex_unlock(&md->buf_mutex); 1121 return -EFAULT; 1122 } 1123 1124 return 0; 1125 } 1126 1127 static int rio_mport_free_dma(struct file *filp, void __user *arg) 1128 { 1129 struct mport_cdev_priv *priv = filp->private_data; 1130 struct mport_dev *md = priv->md; 1131 u64 handle; 1132 int ret = -EFAULT; 1133 struct rio_mport_mapping *map, *_map; 1134 1135 if (copy_from_user(&handle, arg, sizeof(handle))) 1136 return -EFAULT; 1137 rmcd_debug(EXIT, "filp=%p", filp); 1138 1139 mutex_lock(&md->buf_mutex); 1140 list_for_each_entry_safe(map, _map, &md->mappings, node) { 1141 if (map->dir == MAP_DMA && map->phys_addr == handle && 1142 map->filp == filp) { 1143 kref_put(&map->ref, mport_release_mapping); 1144 ret = 0; 1145 break; 1146 } 1147 } 1148 mutex_unlock(&md->buf_mutex); 1149 1150 if (ret == -EFAULT) { 1151 rmcd_debug(DMA, "ERR no matching mapping"); 1152 return ret; 1153 } 1154 1155 return 0; 1156 } 1157 #else 1158 static int rio_mport_transfer_ioctl(struct file *filp, void *arg) 1159 { 1160 return -ENODEV; 1161 } 1162 1163 static int rio_mport_wait_for_async_dma(struct file *filp, void __user *arg) 1164 { 1165 return -ENODEV; 1166 } 1167 1168 static int rio_mport_alloc_dma(struct file *filp, void __user *arg) 1169 { 1170 return -ENODEV; 1171 } 1172 1173 static int rio_mport_free_dma(struct file *filp, void __user *arg) 1174 { 1175 return -ENODEV; 1176 } 1177 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */ 1178 1179 /* 1180 * Inbound/outbound memory mapping functions 1181 */ 1182 1183 static int 1184 rio_mport_create_inbound_mapping(struct mport_dev *md, struct file *filp, 1185 u64 raddr, u64 size, 1186 struct rio_mport_mapping **mapping) 1187 { 1188 struct rio_mport *mport = md->mport; 1189 struct rio_mport_mapping *map; 1190 int ret; 1191 1192 /* rio_map_inb_region() accepts u32 size */ 1193 if (size > 0xffffffff) 1194 return -EINVAL; 1195 1196 map = kzalloc_obj(*map); 1197 if (map == NULL) 1198 return -ENOMEM; 1199 1200 map->virt_addr = dma_alloc_coherent(mport->dev.parent, size, 1201 &map->phys_addr, GFP_KERNEL); 1202 if (map->virt_addr == NULL) { 1203 ret = -ENOMEM; 1204 goto err_dma_alloc; 1205 } 1206 1207 if (raddr == RIO_MAP_ANY_ADDR) 1208 raddr = map->phys_addr; 1209 ret = rio_map_inb_region(mport, map->phys_addr, raddr, (u32)size, 0); 1210 if (ret < 0) 1211 goto err_map_inb; 1212 1213 map->dir = MAP_INBOUND; 1214 map->rio_addr = raddr; 1215 map->size = size; 1216 map->filp = filp; 1217 map->md = md; 1218 kref_init(&map->ref); 1219 mutex_lock(&md->buf_mutex); 1220 list_add_tail(&map->node, &md->mappings); 1221 mutex_unlock(&md->buf_mutex); 1222 *mapping = map; 1223 return 0; 1224 1225 err_map_inb: 1226 dma_free_coherent(mport->dev.parent, size, 1227 map->virt_addr, map->phys_addr); 1228 err_dma_alloc: 1229 kfree(map); 1230 return ret; 1231 } 1232 1233 static int 1234 rio_mport_get_inbound_mapping(struct mport_dev *md, struct file *filp, 1235 u64 raddr, u64 size, 1236 struct rio_mport_mapping **mapping) 1237 { 1238 struct rio_mport_mapping *map; 1239 int err = -ENOMEM; 1240 1241 if (raddr == RIO_MAP_ANY_ADDR) 1242 goto get_new; 1243 1244 mutex_lock(&md->buf_mutex); 1245 list_for_each_entry(map, &md->mappings, node) { 1246 if (map->dir != MAP_INBOUND) 1247 continue; 1248 if (raddr == map->rio_addr && size == map->size) { 1249 /* allow exact match only */ 1250 *mapping = map; 1251 err = 0; 1252 break; 1253 } else if (raddr < (map->rio_addr + map->size - 1) && 1254 (raddr + size) > map->rio_addr) { 1255 err = -EBUSY; 1256 break; 1257 } 1258 } 1259 mutex_unlock(&md->buf_mutex); 1260 1261 if (err != -ENOMEM) 1262 return err; 1263 get_new: 1264 /* not found, create new */ 1265 return rio_mport_create_inbound_mapping(md, filp, raddr, size, mapping); 1266 } 1267 1268 static int rio_mport_map_inbound(struct file *filp, void __user *arg) 1269 { 1270 struct mport_cdev_priv *priv = filp->private_data; 1271 struct mport_dev *md = priv->md; 1272 struct rio_mmap map; 1273 struct rio_mport_mapping *mapping = NULL; 1274 int ret; 1275 1276 if (!md->mport->ops->map_inb) 1277 return -EPROTONOSUPPORT; 1278 if (unlikely(copy_from_user(&map, arg, sizeof(map)))) 1279 return -EFAULT; 1280 1281 rmcd_debug(IBW, "%s filp=%p", dev_name(&priv->md->dev), filp); 1282 1283 ret = rio_mport_get_inbound_mapping(md, filp, map.rio_addr, 1284 map.length, &mapping); 1285 if (ret) 1286 return ret; 1287 1288 map.handle = mapping->phys_addr; 1289 map.rio_addr = mapping->rio_addr; 1290 1291 if (unlikely(copy_to_user(arg, &map, sizeof(map)))) { 1292 /* Delete mapping if it was created by this request */ 1293 if (ret == 0 && mapping->filp == filp) { 1294 mutex_lock(&md->buf_mutex); 1295 kref_put(&mapping->ref, mport_release_mapping); 1296 mutex_unlock(&md->buf_mutex); 1297 } 1298 return -EFAULT; 1299 } 1300 1301 return 0; 1302 } 1303 1304 /* 1305 * rio_mport_inbound_free() - unmap from RapidIO address space and free 1306 * previously allocated inbound DMA coherent buffer 1307 * @priv: driver private data 1308 * @arg: buffer handle returned by allocation routine 1309 */ 1310 static int rio_mport_inbound_free(struct file *filp, void __user *arg) 1311 { 1312 struct mport_cdev_priv *priv = filp->private_data; 1313 struct mport_dev *md = priv->md; 1314 u64 handle; 1315 struct rio_mport_mapping *map, *_map; 1316 1317 rmcd_debug(IBW, "%s filp=%p", dev_name(&priv->md->dev), filp); 1318 1319 if (!md->mport->ops->unmap_inb) 1320 return -EPROTONOSUPPORT; 1321 1322 if (copy_from_user(&handle, arg, sizeof(handle))) 1323 return -EFAULT; 1324 1325 mutex_lock(&md->buf_mutex); 1326 list_for_each_entry_safe(map, _map, &md->mappings, node) { 1327 if (map->dir == MAP_INBOUND && map->phys_addr == handle) { 1328 if (map->filp == filp) { 1329 map->filp = NULL; 1330 kref_put(&map->ref, mport_release_mapping); 1331 } 1332 break; 1333 } 1334 } 1335 mutex_unlock(&md->buf_mutex); 1336 1337 return 0; 1338 } 1339 1340 /* 1341 * maint_port_idx_get() - Get the port index of the mport instance 1342 * @priv: driver private data 1343 * @arg: port index 1344 */ 1345 static int maint_port_idx_get(struct mport_cdev_priv *priv, void __user *arg) 1346 { 1347 struct mport_dev *md = priv->md; 1348 u32 port_idx = md->mport->index; 1349 1350 rmcd_debug(MPORT, "port_index=%d", port_idx); 1351 1352 if (copy_to_user(arg, &port_idx, sizeof(port_idx))) 1353 return -EFAULT; 1354 1355 return 0; 1356 } 1357 1358 static int rio_mport_add_event(struct mport_cdev_priv *priv, 1359 struct rio_event *event) 1360 { 1361 int overflow; 1362 1363 if (!(priv->event_mask & event->header)) 1364 return -EACCES; 1365 1366 spin_lock(&priv->fifo_lock); 1367 overflow = kfifo_avail(&priv->event_fifo) < sizeof(*event) 1368 || kfifo_in(&priv->event_fifo, (unsigned char *)event, 1369 sizeof(*event)) != sizeof(*event); 1370 spin_unlock(&priv->fifo_lock); 1371 1372 wake_up_interruptible(&priv->event_rx_wait); 1373 1374 if (overflow) { 1375 dev_warn(&priv->md->dev, DRV_NAME ": event fifo overflow\n"); 1376 return -EBUSY; 1377 } 1378 1379 return 0; 1380 } 1381 1382 static void rio_mport_doorbell_handler(struct rio_mport *mport, void *dev_id, 1383 u16 src, u16 dst, u16 info) 1384 { 1385 struct mport_dev *data = dev_id; 1386 struct mport_cdev_priv *priv; 1387 struct rio_mport_db_filter *db_filter; 1388 struct rio_event event; 1389 int handled; 1390 1391 event.header = RIO_DOORBELL; 1392 event.u.doorbell.rioid = src; 1393 event.u.doorbell.payload = info; 1394 1395 handled = 0; 1396 spin_lock(&data->db_lock); 1397 list_for_each_entry(db_filter, &data->doorbells, data_node) { 1398 if (((db_filter->filter.rioid == RIO_INVALID_DESTID || 1399 db_filter->filter.rioid == src)) && 1400 info >= db_filter->filter.low && 1401 info <= db_filter->filter.high) { 1402 priv = db_filter->priv; 1403 rio_mport_add_event(priv, &event); 1404 handled = 1; 1405 } 1406 } 1407 spin_unlock(&data->db_lock); 1408 1409 if (!handled) 1410 dev_warn(&data->dev, 1411 "%s: spurious DB received from 0x%x, info=0x%04x\n", 1412 __func__, src, info); 1413 } 1414 1415 static int rio_mport_add_db_filter(struct mport_cdev_priv *priv, 1416 void __user *arg) 1417 { 1418 struct mport_dev *md = priv->md; 1419 struct rio_mport_db_filter *db_filter; 1420 struct rio_doorbell_filter filter; 1421 unsigned long flags; 1422 int ret; 1423 1424 if (copy_from_user(&filter, arg, sizeof(filter))) 1425 return -EFAULT; 1426 1427 if (filter.low > filter.high) 1428 return -EINVAL; 1429 1430 ret = rio_request_inb_dbell(md->mport, md, filter.low, filter.high, 1431 rio_mport_doorbell_handler); 1432 if (ret) { 1433 rmcd_error("%s failed to register IBDB, err=%d", 1434 dev_name(&md->dev), ret); 1435 return ret; 1436 } 1437 1438 db_filter = kzalloc_obj(*db_filter); 1439 if (db_filter == NULL) { 1440 rio_release_inb_dbell(md->mport, filter.low, filter.high); 1441 return -ENOMEM; 1442 } 1443 1444 db_filter->filter = filter; 1445 db_filter->priv = priv; 1446 spin_lock_irqsave(&md->db_lock, flags); 1447 list_add_tail(&db_filter->priv_node, &priv->db_filters); 1448 list_add_tail(&db_filter->data_node, &md->doorbells); 1449 spin_unlock_irqrestore(&md->db_lock, flags); 1450 1451 return 0; 1452 } 1453 1454 static void rio_mport_delete_db_filter(struct rio_mport_db_filter *db_filter) 1455 { 1456 list_del(&db_filter->data_node); 1457 list_del(&db_filter->priv_node); 1458 kfree(db_filter); 1459 } 1460 1461 static int rio_mport_remove_db_filter(struct mport_cdev_priv *priv, 1462 void __user *arg) 1463 { 1464 struct rio_mport_db_filter *db_filter; 1465 struct rio_doorbell_filter filter; 1466 unsigned long flags; 1467 int ret = -EINVAL; 1468 1469 if (copy_from_user(&filter, arg, sizeof(filter))) 1470 return -EFAULT; 1471 1472 if (filter.low > filter.high) 1473 return -EINVAL; 1474 1475 spin_lock_irqsave(&priv->md->db_lock, flags); 1476 list_for_each_entry(db_filter, &priv->db_filters, priv_node) { 1477 if (db_filter->filter.rioid == filter.rioid && 1478 db_filter->filter.low == filter.low && 1479 db_filter->filter.high == filter.high) { 1480 rio_mport_delete_db_filter(db_filter); 1481 ret = 0; 1482 break; 1483 } 1484 } 1485 spin_unlock_irqrestore(&priv->md->db_lock, flags); 1486 1487 if (!ret) 1488 rio_release_inb_dbell(priv->md->mport, filter.low, filter.high); 1489 1490 return ret; 1491 } 1492 1493 static int rio_mport_match_pw(union rio_pw_msg *msg, 1494 struct rio_pw_filter *filter) 1495 { 1496 if ((msg->em.comptag & filter->mask) < filter->low || 1497 (msg->em.comptag & filter->mask) > filter->high) 1498 return 0; 1499 return 1; 1500 } 1501 1502 static int rio_mport_pw_handler(struct rio_mport *mport, void *context, 1503 union rio_pw_msg *msg, int step) 1504 { 1505 struct mport_dev *md = context; 1506 struct mport_cdev_priv *priv; 1507 struct rio_mport_pw_filter *pw_filter; 1508 struct rio_event event; 1509 int handled; 1510 1511 event.header = RIO_PORTWRITE; 1512 memcpy(event.u.portwrite.payload, msg->raw, RIO_PW_MSG_SIZE); 1513 1514 handled = 0; 1515 spin_lock(&md->pw_lock); 1516 list_for_each_entry(pw_filter, &md->portwrites, md_node) { 1517 if (rio_mport_match_pw(msg, &pw_filter->filter)) { 1518 priv = pw_filter->priv; 1519 rio_mport_add_event(priv, &event); 1520 handled = 1; 1521 } 1522 } 1523 spin_unlock(&md->pw_lock); 1524 1525 if (!handled) { 1526 printk_ratelimited(KERN_WARNING DRV_NAME 1527 ": mport%d received spurious PW from 0x%08x\n", 1528 mport->id, msg->em.comptag); 1529 } 1530 1531 return 0; 1532 } 1533 1534 static int rio_mport_add_pw_filter(struct mport_cdev_priv *priv, 1535 void __user *arg) 1536 { 1537 struct mport_dev *md = priv->md; 1538 struct rio_mport_pw_filter *pw_filter; 1539 struct rio_pw_filter filter; 1540 unsigned long flags; 1541 int hadd = 0; 1542 1543 if (copy_from_user(&filter, arg, sizeof(filter))) 1544 return -EFAULT; 1545 1546 pw_filter = kzalloc_obj(*pw_filter); 1547 if (pw_filter == NULL) 1548 return -ENOMEM; 1549 1550 pw_filter->filter = filter; 1551 pw_filter->priv = priv; 1552 spin_lock_irqsave(&md->pw_lock, flags); 1553 if (list_empty(&md->portwrites)) 1554 hadd = 1; 1555 list_add_tail(&pw_filter->priv_node, &priv->pw_filters); 1556 list_add_tail(&pw_filter->md_node, &md->portwrites); 1557 spin_unlock_irqrestore(&md->pw_lock, flags); 1558 1559 if (hadd) { 1560 int ret; 1561 1562 ret = rio_add_mport_pw_handler(md->mport, md, 1563 rio_mport_pw_handler); 1564 if (ret) { 1565 dev_err(&md->dev, 1566 "%s: failed to add IB_PW handler, err=%d\n", 1567 __func__, ret); 1568 return ret; 1569 } 1570 rio_pw_enable(md->mport, 1); 1571 } 1572 1573 return 0; 1574 } 1575 1576 static void rio_mport_delete_pw_filter(struct rio_mport_pw_filter *pw_filter) 1577 { 1578 list_del(&pw_filter->md_node); 1579 list_del(&pw_filter->priv_node); 1580 kfree(pw_filter); 1581 } 1582 1583 static int rio_mport_match_pw_filter(struct rio_pw_filter *a, 1584 struct rio_pw_filter *b) 1585 { 1586 if ((a->mask == b->mask) && (a->low == b->low) && (a->high == b->high)) 1587 return 1; 1588 return 0; 1589 } 1590 1591 static int rio_mport_remove_pw_filter(struct mport_cdev_priv *priv, 1592 void __user *arg) 1593 { 1594 struct mport_dev *md = priv->md; 1595 struct rio_mport_pw_filter *pw_filter; 1596 struct rio_pw_filter filter; 1597 unsigned long flags; 1598 int ret = -EINVAL; 1599 int hdel = 0; 1600 1601 if (copy_from_user(&filter, arg, sizeof(filter))) 1602 return -EFAULT; 1603 1604 spin_lock_irqsave(&md->pw_lock, flags); 1605 list_for_each_entry(pw_filter, &priv->pw_filters, priv_node) { 1606 if (rio_mport_match_pw_filter(&pw_filter->filter, &filter)) { 1607 rio_mport_delete_pw_filter(pw_filter); 1608 ret = 0; 1609 break; 1610 } 1611 } 1612 1613 if (list_empty(&md->portwrites)) 1614 hdel = 1; 1615 spin_unlock_irqrestore(&md->pw_lock, flags); 1616 1617 if (hdel) { 1618 rio_del_mport_pw_handler(md->mport, priv->md, 1619 rio_mport_pw_handler); 1620 rio_pw_enable(md->mport, 0); 1621 } 1622 1623 return ret; 1624 } 1625 1626 /* 1627 * rio_release_dev - release routine for kernel RIO device object 1628 * @dev: kernel device object associated with a RIO device structure 1629 * 1630 * Frees a RIO device struct associated a RIO device struct. 1631 * The RIO device struct is freed. 1632 */ 1633 static void rio_release_dev(struct device *dev) 1634 { 1635 struct rio_dev *rdev; 1636 1637 rdev = to_rio_dev(dev); 1638 pr_info(DRV_PREFIX "%s: %s\n", __func__, rio_name(rdev)); 1639 kfree(rdev); 1640 } 1641 1642 1643 static void rio_release_net(struct device *dev) 1644 { 1645 struct rio_net *net; 1646 1647 net = to_rio_net(dev); 1648 rmcd_debug(RDEV, "net_%d", net->id); 1649 kfree(net); 1650 } 1651 1652 1653 /* 1654 * rio_mport_add_riodev - creates a kernel RIO device object 1655 * 1656 * Allocates a RIO device data structure and initializes required fields based 1657 * on device's configuration space contents. 1658 * If the device has switch capabilities, then a switch specific portion is 1659 * allocated and configured. 1660 */ 1661 static int rio_mport_add_riodev(struct mport_cdev_priv *priv, 1662 void __user *arg) 1663 { 1664 struct mport_dev *md = priv->md; 1665 struct rio_rdev_info dev_info; 1666 struct rio_dev *rdev; 1667 struct rio_switch *rswitch = NULL; 1668 struct rio_mport *mport; 1669 struct device *dev; 1670 size_t size; 1671 u32 rval; 1672 u32 swpinfo = 0; 1673 u16 destid; 1674 u8 hopcount; 1675 int err; 1676 1677 if (copy_from_user(&dev_info, arg, sizeof(dev_info))) 1678 return -EFAULT; 1679 dev_info.name[sizeof(dev_info.name) - 1] = '\0'; 1680 1681 rmcd_debug(RDEV, "name:%s ct:0x%x did:0x%x hc:0x%x", dev_info.name, 1682 dev_info.comptag, dev_info.destid, dev_info.hopcount); 1683 1684 dev = bus_find_device_by_name(&rio_bus_type, NULL, dev_info.name); 1685 if (dev) { 1686 rmcd_debug(RDEV, "device %s already exists", dev_info.name); 1687 put_device(dev); 1688 return -EEXIST; 1689 } 1690 1691 size = sizeof(*rdev); 1692 mport = md->mport; 1693 destid = dev_info.destid; 1694 hopcount = dev_info.hopcount; 1695 1696 if (rio_mport_read_config_32(mport, destid, hopcount, 1697 RIO_PEF_CAR, &rval)) 1698 return -EIO; 1699 1700 if (rval & RIO_PEF_SWITCH) { 1701 rio_mport_read_config_32(mport, destid, hopcount, 1702 RIO_SWP_INFO_CAR, &swpinfo); 1703 size += struct_size(rswitch, nextdev, RIO_GET_TOTAL_PORTS(swpinfo)); 1704 } 1705 1706 rdev = kzalloc(size, GFP_KERNEL); 1707 if (rdev == NULL) 1708 return -ENOMEM; 1709 1710 if (mport->net == NULL) { 1711 struct rio_net *net; 1712 1713 net = rio_alloc_net(mport); 1714 if (!net) { 1715 err = -ENOMEM; 1716 rmcd_debug(RDEV, "failed to allocate net object"); 1717 goto cleanup; 1718 } 1719 1720 net->id = mport->id; 1721 net->hport = mport; 1722 dev_set_name(&net->dev, "rnet_%d", net->id); 1723 net->dev.parent = &mport->dev; 1724 net->dev.release = rio_release_net; 1725 err = rio_add_net(net); 1726 if (err) { 1727 rmcd_debug(RDEV, "failed to register net, err=%d", err); 1728 put_device(&net->dev); 1729 mport->net = NULL; 1730 goto cleanup; 1731 } 1732 } 1733 1734 rdev->net = mport->net; 1735 rdev->pef = rval; 1736 rdev->swpinfo = swpinfo; 1737 rio_mport_read_config_32(mport, destid, hopcount, 1738 RIO_DEV_ID_CAR, &rval); 1739 rdev->did = rval >> 16; 1740 rdev->vid = rval & 0xffff; 1741 rio_mport_read_config_32(mport, destid, hopcount, RIO_DEV_INFO_CAR, 1742 &rdev->device_rev); 1743 rio_mport_read_config_32(mport, destid, hopcount, RIO_ASM_ID_CAR, 1744 &rval); 1745 rdev->asm_did = rval >> 16; 1746 rdev->asm_vid = rval & 0xffff; 1747 rio_mport_read_config_32(mport, destid, hopcount, RIO_ASM_INFO_CAR, 1748 &rval); 1749 rdev->asm_rev = rval >> 16; 1750 1751 if (rdev->pef & RIO_PEF_EXT_FEATURES) { 1752 rdev->efptr = rval & 0xffff; 1753 rdev->phys_efptr = rio_mport_get_physefb(mport, 0, destid, 1754 hopcount, &rdev->phys_rmap); 1755 1756 rdev->em_efptr = rio_mport_get_feature(mport, 0, destid, 1757 hopcount, RIO_EFB_ERR_MGMNT); 1758 } 1759 1760 rio_mport_read_config_32(mport, destid, hopcount, RIO_SRC_OPS_CAR, 1761 &rdev->src_ops); 1762 rio_mport_read_config_32(mport, destid, hopcount, RIO_DST_OPS_CAR, 1763 &rdev->dst_ops); 1764 1765 rdev->comp_tag = dev_info.comptag; 1766 rdev->destid = destid; 1767 /* hopcount is stored as specified by a caller, regardles of EP or SW */ 1768 rdev->hopcount = hopcount; 1769 1770 if (rdev->pef & RIO_PEF_SWITCH) { 1771 rswitch = rdev->rswitch; 1772 rswitch->route_table = NULL; 1773 } 1774 1775 if (strlen(dev_info.name)) 1776 dev_set_name(&rdev->dev, "%s", dev_info.name); 1777 else if (rdev->pef & RIO_PEF_SWITCH) 1778 dev_set_name(&rdev->dev, "%02x:s:%04x", mport->id, 1779 rdev->comp_tag & RIO_CTAG_UDEVID); 1780 else 1781 dev_set_name(&rdev->dev, "%02x:e:%04x", mport->id, 1782 rdev->comp_tag & RIO_CTAG_UDEVID); 1783 1784 INIT_LIST_HEAD(&rdev->net_list); 1785 rdev->dev.parent = &mport->net->dev; 1786 rio_attach_device(rdev); 1787 rdev->dev.release = rio_release_dev; 1788 1789 if (rdev->dst_ops & RIO_DST_OPS_DOORBELL) 1790 rio_init_dbell_res(&rdev->riores[RIO_DOORBELL_RESOURCE], 1791 0, 0xffff); 1792 err = rio_add_device(rdev); 1793 if (err) { 1794 put_device(&rdev->dev); 1795 return err; 1796 } 1797 1798 rio_dev_get(rdev); 1799 1800 return 0; 1801 cleanup: 1802 kfree(rdev); 1803 return err; 1804 } 1805 1806 static int rio_mport_del_riodev(struct mport_cdev_priv *priv, void __user *arg) 1807 { 1808 struct rio_rdev_info dev_info; 1809 struct rio_dev *rdev = NULL; 1810 struct device *dev; 1811 struct rio_mport *mport; 1812 struct rio_net *net; 1813 1814 if (copy_from_user(&dev_info, arg, sizeof(dev_info))) 1815 return -EFAULT; 1816 dev_info.name[sizeof(dev_info.name) - 1] = '\0'; 1817 1818 mport = priv->md->mport; 1819 1820 /* If device name is specified, removal by name has priority */ 1821 if (strlen(dev_info.name)) { 1822 dev = bus_find_device_by_name(&rio_bus_type, NULL, 1823 dev_info.name); 1824 if (dev) 1825 rdev = to_rio_dev(dev); 1826 } else { 1827 do { 1828 rdev = rio_get_comptag(dev_info.comptag, rdev); 1829 if (rdev && rdev->dev.parent == &mport->net->dev && 1830 rdev->destid == dev_info.destid && 1831 rdev->hopcount == dev_info.hopcount) 1832 break; 1833 } while (rdev); 1834 } 1835 1836 if (!rdev) { 1837 rmcd_debug(RDEV, 1838 "device name:%s ct:0x%x did:0x%x hc:0x%x not found", 1839 dev_info.name, dev_info.comptag, dev_info.destid, 1840 dev_info.hopcount); 1841 return -ENODEV; 1842 } 1843 1844 net = rdev->net; 1845 rio_dev_put(rdev); 1846 rio_del_device(rdev, RIO_DEVICE_SHUTDOWN); 1847 1848 if (list_empty(&net->devices)) { 1849 rio_free_net(net); 1850 mport->net = NULL; 1851 } 1852 1853 return 0; 1854 } 1855 1856 /* 1857 * Mport cdev management 1858 */ 1859 1860 /* 1861 * mport_cdev_open() - Open character device (mport) 1862 */ 1863 static int mport_cdev_open(struct inode *inode, struct file *filp) 1864 { 1865 int ret; 1866 int minor = iminor(inode); 1867 struct mport_dev *chdev; 1868 struct mport_cdev_priv *priv; 1869 1870 /* Test for valid device */ 1871 if (minor >= RIO_MAX_MPORTS) { 1872 rmcd_error("Invalid minor device number"); 1873 return -EINVAL; 1874 } 1875 1876 chdev = container_of(inode->i_cdev, struct mport_dev, cdev); 1877 1878 rmcd_debug(INIT, "%s filp=%p", dev_name(&chdev->dev), filp); 1879 1880 if (atomic_read(&chdev->active) == 0) 1881 return -ENODEV; 1882 1883 get_device(&chdev->dev); 1884 1885 priv = kzalloc_obj(*priv); 1886 if (!priv) { 1887 put_device(&chdev->dev); 1888 return -ENOMEM; 1889 } 1890 1891 priv->md = chdev; 1892 1893 INIT_LIST_HEAD(&priv->db_filters); 1894 INIT_LIST_HEAD(&priv->pw_filters); 1895 spin_lock_init(&priv->fifo_lock); 1896 init_waitqueue_head(&priv->event_rx_wait); 1897 ret = kfifo_alloc(&priv->event_fifo, 1898 sizeof(struct rio_event) * MPORT_EVENT_DEPTH, 1899 GFP_KERNEL); 1900 if (ret < 0) { 1901 put_device(&chdev->dev); 1902 dev_err(&chdev->dev, DRV_NAME ": kfifo_alloc failed\n"); 1903 ret = -ENOMEM; 1904 goto err_fifo; 1905 } 1906 1907 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 1908 INIT_LIST_HEAD(&priv->async_list); 1909 spin_lock_init(&priv->req_lock); 1910 mutex_init(&priv->dma_lock); 1911 #endif 1912 mutex_lock(&chdev->file_mutex); 1913 list_add_tail(&priv->list, &chdev->file_list); 1914 mutex_unlock(&chdev->file_mutex); 1915 1916 filp->private_data = priv; 1917 goto out; 1918 err_fifo: 1919 kfree(priv); 1920 out: 1921 return ret; 1922 } 1923 1924 static int mport_cdev_fasync(int fd, struct file *filp, int mode) 1925 { 1926 struct mport_cdev_priv *priv = filp->private_data; 1927 1928 return fasync_helper(fd, filp, mode, &priv->async_queue); 1929 } 1930 1931 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 1932 static void mport_cdev_release_dma(struct file *filp) 1933 { 1934 struct mport_cdev_priv *priv = filp->private_data; 1935 struct mport_dev *md; 1936 struct mport_dma_req *req, *req_next; 1937 unsigned long tmo = msecs_to_jiffies(dma_timeout); 1938 long wret; 1939 LIST_HEAD(list); 1940 1941 rmcd_debug(EXIT, "from filp=%p %s(%d)", 1942 filp, current->comm, task_pid_nr(current)); 1943 1944 if (!priv->dmach) { 1945 rmcd_debug(EXIT, "No DMA channel for filp=%p", filp); 1946 return; 1947 } 1948 1949 md = priv->md; 1950 1951 spin_lock(&priv->req_lock); 1952 if (!list_empty(&priv->async_list)) { 1953 rmcd_debug(EXIT, "async list not empty filp=%p %s(%d)", 1954 filp, current->comm, task_pid_nr(current)); 1955 list_splice_init(&priv->async_list, &list); 1956 } 1957 spin_unlock(&priv->req_lock); 1958 1959 if (!list_empty(&list)) { 1960 rmcd_debug(EXIT, "temp list not empty"); 1961 list_for_each_entry_safe(req, req_next, &list, node) { 1962 rmcd_debug(EXIT, "free req->filp=%p cookie=%d compl=%s", 1963 req->filp, req->cookie, 1964 completion_done(&req->req_comp)?"yes":"no"); 1965 list_del(&req->node); 1966 kref_put(&req->refcount, dma_req_free); 1967 } 1968 } 1969 1970 put_dma_channel(priv); 1971 wret = wait_for_completion_interruptible_timeout(&priv->comp, tmo); 1972 1973 if (wret <= 0) { 1974 rmcd_error("%s(%d) failed waiting for DMA release err=%ld", 1975 current->comm, task_pid_nr(current), wret); 1976 } 1977 1978 if (priv->dmach != priv->md->dma_chan) { 1979 rmcd_debug(EXIT, "Release DMA channel for filp=%p %s(%d)", 1980 filp, current->comm, task_pid_nr(current)); 1981 rio_release_dma(priv->dmach); 1982 } else { 1983 rmcd_debug(EXIT, "Adjust default DMA channel refcount"); 1984 kref_put(&md->dma_ref, mport_release_def_dma); 1985 } 1986 1987 priv->dmach = NULL; 1988 } 1989 #else 1990 #define mport_cdev_release_dma(priv) do {} while (0) 1991 #endif 1992 1993 /* 1994 * mport_cdev_release() - Release character device 1995 */ 1996 static int mport_cdev_release(struct inode *inode, struct file *filp) 1997 { 1998 struct mport_cdev_priv *priv = filp->private_data; 1999 struct mport_dev *chdev; 2000 struct rio_mport_pw_filter *pw_filter, *pw_filter_next; 2001 struct rio_mport_db_filter *db_filter, *db_filter_next; 2002 struct rio_mport_mapping *map, *_map; 2003 unsigned long flags; 2004 2005 rmcd_debug(EXIT, "%s filp=%p", dev_name(&priv->md->dev), filp); 2006 2007 chdev = priv->md; 2008 mport_cdev_release_dma(filp); 2009 2010 priv->event_mask = 0; 2011 2012 spin_lock_irqsave(&chdev->pw_lock, flags); 2013 if (!list_empty(&priv->pw_filters)) { 2014 list_for_each_entry_safe(pw_filter, pw_filter_next, 2015 &priv->pw_filters, priv_node) 2016 rio_mport_delete_pw_filter(pw_filter); 2017 } 2018 spin_unlock_irqrestore(&chdev->pw_lock, flags); 2019 2020 spin_lock_irqsave(&chdev->db_lock, flags); 2021 list_for_each_entry_safe(db_filter, db_filter_next, 2022 &priv->db_filters, priv_node) { 2023 rio_mport_delete_db_filter(db_filter); 2024 } 2025 spin_unlock_irqrestore(&chdev->db_lock, flags); 2026 2027 kfifo_free(&priv->event_fifo); 2028 2029 mutex_lock(&chdev->buf_mutex); 2030 list_for_each_entry_safe(map, _map, &chdev->mappings, node) { 2031 if (map->filp == filp) { 2032 rmcd_debug(EXIT, "release mapping %p filp=%p", 2033 map->virt_addr, filp); 2034 kref_put(&map->ref, mport_release_mapping); 2035 } 2036 } 2037 mutex_unlock(&chdev->buf_mutex); 2038 2039 mport_cdev_fasync(-1, filp, 0); 2040 filp->private_data = NULL; 2041 mutex_lock(&chdev->file_mutex); 2042 list_del(&priv->list); 2043 mutex_unlock(&chdev->file_mutex); 2044 put_device(&chdev->dev); 2045 kfree(priv); 2046 return 0; 2047 } 2048 2049 /* 2050 * mport_cdev_ioctl() - IOCTLs for character device 2051 */ 2052 static long mport_cdev_ioctl(struct file *filp, 2053 unsigned int cmd, unsigned long arg) 2054 { 2055 int err = -EINVAL; 2056 struct mport_cdev_priv *data = filp->private_data; 2057 struct mport_dev *md = data->md; 2058 2059 if (atomic_read(&md->active) == 0) 2060 return -ENODEV; 2061 2062 switch (cmd) { 2063 case RIO_MPORT_MAINT_READ_LOCAL: 2064 return rio_mport_maint_rd(data, (void __user *)arg, 1); 2065 case RIO_MPORT_MAINT_WRITE_LOCAL: 2066 return rio_mport_maint_wr(data, (void __user *)arg, 1); 2067 case RIO_MPORT_MAINT_READ_REMOTE: 2068 return rio_mport_maint_rd(data, (void __user *)arg, 0); 2069 case RIO_MPORT_MAINT_WRITE_REMOTE: 2070 return rio_mport_maint_wr(data, (void __user *)arg, 0); 2071 case RIO_MPORT_MAINT_HDID_SET: 2072 return maint_hdid_set(data, (void __user *)arg); 2073 case RIO_MPORT_MAINT_COMPTAG_SET: 2074 return maint_comptag_set(data, (void __user *)arg); 2075 case RIO_MPORT_MAINT_PORT_IDX_GET: 2076 return maint_port_idx_get(data, (void __user *)arg); 2077 case RIO_MPORT_GET_PROPERTIES: 2078 md->properties.hdid = md->mport->host_deviceid; 2079 if (copy_to_user((void __user *)arg, &(md->properties), 2080 sizeof(md->properties))) 2081 return -EFAULT; 2082 return 0; 2083 case RIO_ENABLE_DOORBELL_RANGE: 2084 return rio_mport_add_db_filter(data, (void __user *)arg); 2085 case RIO_DISABLE_DOORBELL_RANGE: 2086 return rio_mport_remove_db_filter(data, (void __user *)arg); 2087 case RIO_ENABLE_PORTWRITE_RANGE: 2088 return rio_mport_add_pw_filter(data, (void __user *)arg); 2089 case RIO_DISABLE_PORTWRITE_RANGE: 2090 return rio_mport_remove_pw_filter(data, (void __user *)arg); 2091 case RIO_SET_EVENT_MASK: 2092 data->event_mask = (u32)arg; 2093 return 0; 2094 case RIO_GET_EVENT_MASK: 2095 if (copy_to_user((void __user *)arg, &data->event_mask, 2096 sizeof(u32))) 2097 return -EFAULT; 2098 return 0; 2099 case RIO_MAP_OUTBOUND: 2100 return rio_mport_obw_map(filp, (void __user *)arg); 2101 case RIO_MAP_INBOUND: 2102 return rio_mport_map_inbound(filp, (void __user *)arg); 2103 case RIO_UNMAP_OUTBOUND: 2104 return rio_mport_obw_free(filp, (void __user *)arg); 2105 case RIO_UNMAP_INBOUND: 2106 return rio_mport_inbound_free(filp, (void __user *)arg); 2107 case RIO_ALLOC_DMA: 2108 return rio_mport_alloc_dma(filp, (void __user *)arg); 2109 case RIO_FREE_DMA: 2110 return rio_mport_free_dma(filp, (void __user *)arg); 2111 case RIO_WAIT_FOR_ASYNC: 2112 return rio_mport_wait_for_async_dma(filp, (void __user *)arg); 2113 case RIO_TRANSFER: 2114 return rio_mport_transfer_ioctl(filp, (void __user *)arg); 2115 case RIO_DEV_ADD: 2116 return rio_mport_add_riodev(data, (void __user *)arg); 2117 case RIO_DEV_DEL: 2118 return rio_mport_del_riodev(data, (void __user *)arg); 2119 default: 2120 break; 2121 } 2122 2123 return err; 2124 } 2125 2126 /* 2127 * mport_release_mapping - free mapping resources and info structure 2128 * @ref: a pointer to the kref within struct rio_mport_mapping 2129 * 2130 * NOTE: Shall be called while holding buf_mutex. 2131 */ 2132 static void mport_release_mapping(struct kref *ref) 2133 { 2134 struct rio_mport_mapping *map = 2135 container_of(ref, struct rio_mport_mapping, ref); 2136 struct rio_mport *mport = map->md->mport; 2137 2138 rmcd_debug(MMAP, "type %d mapping @ %p (phys = %pad) for %s", 2139 map->dir, map->virt_addr, 2140 &map->phys_addr, mport->name); 2141 2142 list_del(&map->node); 2143 2144 switch (map->dir) { 2145 case MAP_INBOUND: 2146 rio_unmap_inb_region(mport, map->phys_addr); 2147 fallthrough; 2148 case MAP_DMA: 2149 dma_free_coherent(mport->dev.parent, map->size, 2150 map->virt_addr, map->phys_addr); 2151 break; 2152 case MAP_OUTBOUND: 2153 rio_unmap_outb_region(mport, map->rioid, map->rio_addr); 2154 break; 2155 } 2156 kfree(map); 2157 } 2158 2159 static void mport_mm_open(struct vm_area_struct *vma) 2160 { 2161 struct rio_mport_mapping *map = vma->vm_private_data; 2162 2163 rmcd_debug(MMAP, "%pad", &map->phys_addr); 2164 kref_get(&map->ref); 2165 } 2166 2167 static void mport_mm_close(struct vm_area_struct *vma) 2168 { 2169 struct rio_mport_mapping *map = vma->vm_private_data; 2170 2171 rmcd_debug(MMAP, "%pad", &map->phys_addr); 2172 mutex_lock(&map->md->buf_mutex); 2173 kref_put(&map->ref, mport_release_mapping); 2174 mutex_unlock(&map->md->buf_mutex); 2175 } 2176 2177 static const struct vm_operations_struct vm_ops = { 2178 .open = mport_mm_open, 2179 .close = mport_mm_close, 2180 }; 2181 2182 static int mport_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 2183 { 2184 struct mport_cdev_priv *priv = filp->private_data; 2185 struct mport_dev *md; 2186 size_t size = vma->vm_end - vma->vm_start; 2187 dma_addr_t baddr; 2188 unsigned long offset; 2189 int found = 0, ret; 2190 struct rio_mport_mapping *map; 2191 2192 rmcd_debug(MMAP, "0x%x bytes at offset 0x%lx", 2193 (unsigned int)size, vma->vm_pgoff); 2194 2195 md = priv->md; 2196 baddr = ((dma_addr_t)vma->vm_pgoff << PAGE_SHIFT); 2197 2198 mutex_lock(&md->buf_mutex); 2199 list_for_each_entry(map, &md->mappings, node) { 2200 if (baddr >= map->phys_addr && 2201 baddr < (map->phys_addr + map->size)) { 2202 found = 1; 2203 break; 2204 } 2205 } 2206 mutex_unlock(&md->buf_mutex); 2207 2208 if (!found) 2209 return -ENOMEM; 2210 2211 offset = baddr - map->phys_addr; 2212 2213 if (size + offset > map->size) 2214 return -EINVAL; 2215 2216 vma->vm_pgoff = offset >> PAGE_SHIFT; 2217 rmcd_debug(MMAP, "MMAP adjusted offset = 0x%lx", vma->vm_pgoff); 2218 2219 if (map->dir == MAP_INBOUND || map->dir == MAP_DMA) 2220 ret = dma_mmap_coherent(md->mport->dev.parent, vma, 2221 map->virt_addr, map->phys_addr, map->size); 2222 else if (map->dir == MAP_OUTBOUND) { 2223 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 2224 ret = vm_iomap_memory(vma, map->phys_addr, map->size); 2225 } else { 2226 rmcd_error("Attempt to mmap unsupported mapping type"); 2227 ret = -EIO; 2228 } 2229 2230 if (!ret) { 2231 vma->vm_private_data = map; 2232 vma->vm_ops = &vm_ops; 2233 mport_mm_open(vma); 2234 } else { 2235 rmcd_error("MMAP exit with err=%d", ret); 2236 } 2237 2238 return ret; 2239 } 2240 2241 static __poll_t mport_cdev_poll(struct file *filp, poll_table *wait) 2242 { 2243 struct mport_cdev_priv *priv = filp->private_data; 2244 2245 poll_wait(filp, &priv->event_rx_wait, wait); 2246 if (kfifo_len(&priv->event_fifo)) 2247 return EPOLLIN | EPOLLRDNORM; 2248 2249 return 0; 2250 } 2251 2252 static ssize_t mport_read(struct file *filp, char __user *buf, size_t count, 2253 loff_t *ppos) 2254 { 2255 struct mport_cdev_priv *priv = filp->private_data; 2256 int copied; 2257 ssize_t ret; 2258 2259 if (!count) 2260 return 0; 2261 2262 if (kfifo_is_empty(&priv->event_fifo) && 2263 (filp->f_flags & O_NONBLOCK)) 2264 return -EAGAIN; 2265 2266 if (count % sizeof(struct rio_event)) 2267 return -EINVAL; 2268 2269 ret = wait_event_interruptible(priv->event_rx_wait, 2270 kfifo_len(&priv->event_fifo) != 0); 2271 if (ret) 2272 return ret; 2273 2274 while (ret < count) { 2275 if (kfifo_to_user(&priv->event_fifo, buf, 2276 sizeof(struct rio_event), &copied)) 2277 return -EFAULT; 2278 ret += copied; 2279 buf += copied; 2280 } 2281 2282 return ret; 2283 } 2284 2285 static ssize_t mport_write(struct file *filp, const char __user *buf, 2286 size_t count, loff_t *ppos) 2287 { 2288 struct mport_cdev_priv *priv = filp->private_data; 2289 struct rio_mport *mport = priv->md->mport; 2290 struct rio_event event; 2291 int len, ret; 2292 2293 if (!count) 2294 return 0; 2295 2296 if (count % sizeof(event)) 2297 return -EINVAL; 2298 2299 len = 0; 2300 while ((count - len) >= (int)sizeof(event)) { 2301 if (copy_from_user(&event, buf, sizeof(event))) 2302 return -EFAULT; 2303 2304 if (event.header != RIO_DOORBELL) 2305 return -EINVAL; 2306 2307 ret = rio_mport_send_doorbell(mport, 2308 event.u.doorbell.rioid, 2309 event.u.doorbell.payload); 2310 if (ret < 0) 2311 return ret; 2312 2313 len += sizeof(event); 2314 buf += sizeof(event); 2315 } 2316 2317 return len; 2318 } 2319 2320 static const struct file_operations mport_fops = { 2321 .owner = THIS_MODULE, 2322 .open = mport_cdev_open, 2323 .release = mport_cdev_release, 2324 .poll = mport_cdev_poll, 2325 .read = mport_read, 2326 .write = mport_write, 2327 .mmap = mport_cdev_mmap, 2328 .fasync = mport_cdev_fasync, 2329 .unlocked_ioctl = mport_cdev_ioctl 2330 }; 2331 2332 /* 2333 * Character device management 2334 */ 2335 2336 static void mport_device_release(struct device *dev) 2337 { 2338 struct mport_dev *md; 2339 2340 rmcd_debug(EXIT, "%s", dev_name(dev)); 2341 md = container_of(dev, struct mport_dev, dev); 2342 kfree(md); 2343 } 2344 2345 /* 2346 * mport_cdev_add() - Create mport_dev from rio_mport 2347 * @mport: RapidIO master port 2348 */ 2349 static struct mport_dev *mport_cdev_add(struct rio_mport *mport) 2350 { 2351 int ret = 0; 2352 struct mport_dev *md; 2353 struct rio_mport_attr attr; 2354 2355 md = kzalloc_obj(*md); 2356 if (!md) { 2357 rmcd_error("Unable allocate a device object"); 2358 return NULL; 2359 } 2360 2361 md->mport = mport; 2362 mutex_init(&md->buf_mutex); 2363 mutex_init(&md->file_mutex); 2364 INIT_LIST_HEAD(&md->file_list); 2365 2366 device_initialize(&md->dev); 2367 md->dev.devt = MKDEV(MAJOR(dev_number), mport->id); 2368 md->dev.class = &dev_class; 2369 md->dev.parent = &mport->dev; 2370 md->dev.release = mport_device_release; 2371 dev_set_name(&md->dev, DEV_NAME "%d", mport->id); 2372 atomic_set(&md->active, 1); 2373 2374 cdev_init(&md->cdev, &mport_fops); 2375 md->cdev.owner = THIS_MODULE; 2376 2377 INIT_LIST_HEAD(&md->doorbells); 2378 spin_lock_init(&md->db_lock); 2379 INIT_LIST_HEAD(&md->portwrites); 2380 spin_lock_init(&md->pw_lock); 2381 INIT_LIST_HEAD(&md->mappings); 2382 2383 md->properties.id = mport->id; 2384 md->properties.sys_size = mport->sys_size; 2385 md->properties.hdid = mport->host_deviceid; 2386 md->properties.index = mport->index; 2387 2388 /* The transfer_mode property will be returned through mport query 2389 * interface 2390 */ 2391 #ifdef CONFIG_FSL_RIO /* for now: only on Freescale's SoCs */ 2392 md->properties.transfer_mode |= RIO_TRANSFER_MODE_MAPPED; 2393 #else 2394 md->properties.transfer_mode |= RIO_TRANSFER_MODE_TRANSFER; 2395 #endif 2396 2397 ret = cdev_device_add(&md->cdev, &md->dev); 2398 if (ret) { 2399 rmcd_error("Failed to register mport %d (err=%d)", 2400 mport->id, ret); 2401 goto err_cdev; 2402 } 2403 ret = rio_query_mport(mport, &attr); 2404 if (!ret) { 2405 md->properties.flags = attr.flags; 2406 md->properties.link_speed = attr.link_speed; 2407 md->properties.link_width = attr.link_width; 2408 md->properties.dma_max_sge = attr.dma_max_sge; 2409 md->properties.dma_max_size = attr.dma_max_size; 2410 md->properties.dma_align = attr.dma_align; 2411 md->properties.cap_sys_size = 0; 2412 md->properties.cap_transfer_mode = 0; 2413 md->properties.cap_addr_size = 0; 2414 } else 2415 pr_info(DRV_PREFIX "Failed to obtain info for %s cdev(%d:%d)\n", 2416 mport->name, MAJOR(dev_number), mport->id); 2417 2418 mutex_lock(&mport_devs_lock); 2419 list_add_tail(&md->node, &mport_devs); 2420 mutex_unlock(&mport_devs_lock); 2421 2422 pr_info(DRV_PREFIX "Added %s cdev(%d:%d)\n", 2423 mport->name, MAJOR(dev_number), mport->id); 2424 2425 return md; 2426 2427 err_cdev: 2428 put_device(&md->dev); 2429 return NULL; 2430 } 2431 2432 /* 2433 * mport_cdev_terminate_dma() - Stop all active DMA data transfers and release 2434 * associated DMA channels. 2435 */ 2436 static void mport_cdev_terminate_dma(struct mport_dev *md) 2437 { 2438 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 2439 struct mport_cdev_priv *client; 2440 2441 rmcd_debug(DMA, "%s", dev_name(&md->dev)); 2442 2443 mutex_lock(&md->file_mutex); 2444 list_for_each_entry(client, &md->file_list, list) { 2445 if (client->dmach) { 2446 dmaengine_terminate_all(client->dmach); 2447 rio_release_dma(client->dmach); 2448 } 2449 } 2450 mutex_unlock(&md->file_mutex); 2451 2452 if (md->dma_chan) { 2453 dmaengine_terminate_all(md->dma_chan); 2454 rio_release_dma(md->dma_chan); 2455 md->dma_chan = NULL; 2456 } 2457 #endif 2458 } 2459 2460 2461 /* 2462 * mport_cdev_kill_fasync() - Send SIGIO signal to all processes with open 2463 * mport_cdev files. 2464 */ 2465 static int mport_cdev_kill_fasync(struct mport_dev *md) 2466 { 2467 unsigned int files = 0; 2468 struct mport_cdev_priv *client; 2469 2470 mutex_lock(&md->file_mutex); 2471 list_for_each_entry(client, &md->file_list, list) { 2472 if (client->async_queue) 2473 kill_fasync(&client->async_queue, SIGIO, POLL_HUP); 2474 files++; 2475 } 2476 mutex_unlock(&md->file_mutex); 2477 return files; 2478 } 2479 2480 /* 2481 * mport_cdev_remove() - Remove mport character device 2482 * @dev: Mport device to remove 2483 */ 2484 static void mport_cdev_remove(struct mport_dev *md) 2485 { 2486 struct rio_mport_mapping *map, *_map; 2487 2488 rmcd_debug(EXIT, "Remove %s cdev", md->mport->name); 2489 atomic_set(&md->active, 0); 2490 mport_cdev_terminate_dma(md); 2491 rio_del_mport_pw_handler(md->mport, md, rio_mport_pw_handler); 2492 cdev_device_del(&md->cdev, &md->dev); 2493 mport_cdev_kill_fasync(md); 2494 2495 /* TODO: do we need to give clients some time to close file 2496 * descriptors? Simple wait for XX, or kref? 2497 */ 2498 2499 /* 2500 * Release DMA buffers allocated for the mport device. 2501 * Disable associated inbound Rapidio requests mapping if applicable. 2502 */ 2503 mutex_lock(&md->buf_mutex); 2504 list_for_each_entry_safe(map, _map, &md->mappings, node) { 2505 kref_put(&map->ref, mport_release_mapping); 2506 } 2507 mutex_unlock(&md->buf_mutex); 2508 2509 if (!list_empty(&md->mappings)) 2510 rmcd_warn("WARNING: %s pending mappings on removal", 2511 md->mport->name); 2512 2513 rio_release_inb_dbell(md->mport, 0, 0x0fff); 2514 2515 put_device(&md->dev); 2516 } 2517 2518 /* 2519 * RIO rio_mport_interface driver 2520 */ 2521 2522 /* 2523 * mport_add_mport() - Add rio_mport from LDM device struct 2524 * @dev: Linux device model struct 2525 */ 2526 static int mport_add_mport(struct device *dev) 2527 { 2528 struct rio_mport *mport = NULL; 2529 struct mport_dev *chdev = NULL; 2530 2531 mport = to_rio_mport(dev); 2532 if (!mport) 2533 return -ENODEV; 2534 2535 chdev = mport_cdev_add(mport); 2536 if (!chdev) 2537 return -ENODEV; 2538 2539 return 0; 2540 } 2541 2542 /* 2543 * mport_remove_mport() - Remove rio_mport from global list 2544 * TODO remove device from global mport_dev list 2545 */ 2546 static void mport_remove_mport(struct device *dev) 2547 { 2548 struct rio_mport *mport = NULL; 2549 struct mport_dev *chdev; 2550 int found = 0; 2551 2552 mport = to_rio_mport(dev); 2553 rmcd_debug(EXIT, "Remove %s", mport->name); 2554 2555 mutex_lock(&mport_devs_lock); 2556 list_for_each_entry(chdev, &mport_devs, node) { 2557 if (chdev->mport->id == mport->id) { 2558 atomic_set(&chdev->active, 0); 2559 list_del(&chdev->node); 2560 found = 1; 2561 break; 2562 } 2563 } 2564 mutex_unlock(&mport_devs_lock); 2565 2566 if (found) 2567 mport_cdev_remove(chdev); 2568 } 2569 2570 /* the rio_mport_interface is used to handle local mport devices */ 2571 static struct class_interface rio_mport_interface __refdata = { 2572 .class = &rio_mport_class, 2573 .add_dev = mport_add_mport, 2574 .remove_dev = mport_remove_mport, 2575 }; 2576 2577 /* 2578 * Linux kernel module 2579 */ 2580 2581 /* 2582 * mport_init - Driver module loading 2583 */ 2584 static int __init mport_init(void) 2585 { 2586 int ret; 2587 2588 /* Create device class needed by udev */ 2589 ret = class_register(&dev_class); 2590 if (ret) { 2591 rmcd_error("Unable to create " DRV_NAME " class"); 2592 return ret; 2593 } 2594 2595 ret = alloc_chrdev_region(&dev_number, 0, RIO_MAX_MPORTS, DRV_NAME); 2596 if (ret < 0) 2597 goto err_chr; 2598 2599 rmcd_debug(INIT, "Registered class with major=%d", MAJOR(dev_number)); 2600 2601 /* Register to rio_mport_interface */ 2602 ret = class_interface_register(&rio_mport_interface); 2603 if (ret) { 2604 rmcd_error("class_interface_register() failed, err=%d", ret); 2605 goto err_cli; 2606 } 2607 2608 return 0; 2609 2610 err_cli: 2611 unregister_chrdev_region(dev_number, RIO_MAX_MPORTS); 2612 err_chr: 2613 class_unregister(&dev_class); 2614 return ret; 2615 } 2616 2617 /** 2618 * mport_exit - Driver module unloading 2619 */ 2620 static void __exit mport_exit(void) 2621 { 2622 class_interface_unregister(&rio_mport_interface); 2623 class_unregister(&dev_class); 2624 unregister_chrdev_region(dev_number, RIO_MAX_MPORTS); 2625 } 2626 2627 module_init(mport_init); 2628 module_exit(mport_exit); 2629