1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/drivers/misc/xillybus_core.c 4 * 5 * Copyright 2011 Xillybus Ltd, http://xillybus.com 6 * 7 * Driver for the Xillybus FPGA/host framework. 8 * 9 * This driver interfaces with a special IP core in an FPGA, setting up 10 * a pipe between a hardware FIFO in the programmable logic and a device 11 * file in the host. The number of such pipes and their attributes are 12 * set up on the logic. This driver detects these automatically and 13 * creates the device files accordingly. 14 */ 15 16 #include <linux/list.h> 17 #include <linux/device.h> 18 #include <linux/module.h> 19 #include <linux/io.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/interrupt.h> 22 #include <linux/sched.h> 23 #include <linux/fs.h> 24 #include <linux/spinlock.h> 25 #include <linux/mutex.h> 26 #include <linux/crc32.h> 27 #include <linux/poll.h> 28 #include <linux/delay.h> 29 #include <linux/slab.h> 30 #include <linux/workqueue.h> 31 #include "xillybus.h" 32 #include "xillybus_class.h" 33 34 MODULE_DESCRIPTION("Xillybus core functions"); 35 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); 36 MODULE_ALIAS("xillybus_core"); 37 MODULE_LICENSE("GPL v2"); 38 39 /* General timeout is 100 ms, rx timeout is 10 ms */ 40 #define XILLY_RX_TIMEOUT (10*HZ/1000) 41 #define XILLY_TIMEOUT (100*HZ/1000) 42 43 #define fpga_msg_ctrl_reg 0x0008 44 #define fpga_dma_control_reg 0x0020 45 #define fpga_dma_bufno_reg 0x0024 46 #define fpga_dma_bufaddr_lowaddr_reg 0x0028 47 #define fpga_dma_bufaddr_highaddr_reg 0x002c 48 #define fpga_buf_ctrl_reg 0x0030 49 #define fpga_buf_offset_reg 0x0034 50 #define fpga_endian_reg 0x0040 51 52 #define XILLYMSG_OPCODE_RELEASEBUF 1 53 #define XILLYMSG_OPCODE_QUIESCEACK 2 54 #define XILLYMSG_OPCODE_FIFOEOF 3 55 #define XILLYMSG_OPCODE_FATAL_ERROR 4 56 #define XILLYMSG_OPCODE_NONEMPTY 5 57 58 static const char xillyname[] = "xillybus"; 59 60 static struct workqueue_struct *xillybus_wq; 61 62 /* 63 * Locking scheme: Mutexes protect invocations of character device methods. 64 * If both locks are taken, wr_mutex is taken first, rd_mutex second. 65 * 66 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the 67 * buffers' end_offset fields against changes made by IRQ handler (and in 68 * theory, other file request handlers, but the mutex handles that). Nothing 69 * else. 70 * They are held for short direct memory manipulations. Needless to say, 71 * no mutex locking is allowed when a spinlock is held. 72 * 73 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. 74 * 75 * register_mutex is endpoint-specific, and is held when non-atomic 76 * register operations are performed. wr_mutex and rd_mutex may be 77 * held when register_mutex is taken, but none of the spinlocks. Note that 78 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes 79 * which are unrelated to buf_offset_reg, since they are harmless. 80 * 81 * Blocking on the wait queues is allowed with mutexes held, but not with 82 * spinlocks. 83 * 84 * Only interruptible blocking is allowed on mutexes and wait queues. 85 * 86 * All in all, the locking order goes (with skips allowed, of course): 87 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock 88 */ 89 90 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) 91 { 92 int opcode; 93 int msg_channel, msg_bufno, msg_data, msg_dir; 94 95 opcode = (buf[0] >> 24) & 0xff; 96 msg_dir = buf[0] & 1; 97 msg_channel = (buf[0] >> 1) & 0x7ff; 98 msg_bufno = (buf[0] >> 12) & 0x3ff; 99 msg_data = buf[1] & 0xfffffff; 100 101 dev_warn(endpoint->dev, 102 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", 103 opcode, msg_channel, msg_dir, msg_bufno, msg_data); 104 } 105 106 /* 107 * xillybus_isr assumes the interrupt is allocated exclusively to it, 108 * which is the natural case MSI and several other hardware-oriented 109 * interrupts. Sharing is not allowed. 110 */ 111 112 irqreturn_t xillybus_isr(int irq, void *data) 113 { 114 struct xilly_endpoint *ep = data; 115 u32 *buf; 116 unsigned int buf_size; 117 int i; 118 int opcode; 119 unsigned int msg_channel, msg_bufno, msg_data, msg_dir; 120 struct xilly_channel *channel; 121 122 buf = ep->msgbuf_addr; 123 buf_size = ep->msg_buf_size/sizeof(u32); 124 125 dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr, 126 ep->msg_buf_size, DMA_FROM_DEVICE); 127 128 for (i = 0; i < buf_size; i += 2) { 129 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { 130 malformed_message(ep, &buf[i]); 131 dev_warn(ep->dev, 132 "Sending a NACK on counter %x (instead of %x) on entry %d\n", 133 ((buf[i+1] >> 28) & 0xf), 134 ep->msg_counter, 135 i/2); 136 137 if (++ep->failed_messages > 10) { 138 dev_err(ep->dev, 139 "Lost sync with interrupt messages. Stopping.\n"); 140 } else { 141 dma_sync_single_for_device(ep->dev, 142 ep->msgbuf_dma_addr, 143 ep->msg_buf_size, 144 DMA_FROM_DEVICE); 145 146 iowrite32(0x01, /* Message NACK */ 147 ep->registers + fpga_msg_ctrl_reg); 148 } 149 return IRQ_HANDLED; 150 } else if (buf[i] & (1 << 22)) /* Last message */ 151 break; 152 } 153 154 if (i >= buf_size) { 155 dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); 156 return IRQ_HANDLED; 157 } 158 159 buf_size = i + 2; 160 161 for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ 162 opcode = (buf[i] >> 24) & 0xff; 163 164 msg_dir = buf[i] & 1; 165 msg_channel = (buf[i] >> 1) & 0x7ff; 166 msg_bufno = (buf[i] >> 12) & 0x3ff; 167 msg_data = buf[i+1] & 0xfffffff; 168 169 switch (opcode) { 170 case XILLYMSG_OPCODE_RELEASEBUF: 171 if ((msg_channel > ep->num_channels) || 172 (msg_channel == 0)) { 173 malformed_message(ep, &buf[i]); 174 break; 175 } 176 177 channel = ep->channels[msg_channel]; 178 179 if (msg_dir) { /* Write channel */ 180 if (msg_bufno >= channel->num_wr_buffers) { 181 malformed_message(ep, &buf[i]); 182 break; 183 } 184 spin_lock(&channel->wr_spinlock); 185 channel->wr_buffers[msg_bufno]->end_offset = 186 msg_data; 187 channel->wr_fpga_buf_idx = msg_bufno; 188 channel->wr_empty = 0; 189 channel->wr_sleepy = 0; 190 spin_unlock(&channel->wr_spinlock); 191 192 wake_up_interruptible(&channel->wr_wait); 193 194 } else { 195 /* Read channel */ 196 197 if (msg_bufno >= channel->num_rd_buffers) { 198 malformed_message(ep, &buf[i]); 199 break; 200 } 201 202 spin_lock(&channel->rd_spinlock); 203 channel->rd_fpga_buf_idx = msg_bufno; 204 channel->rd_full = 0; 205 spin_unlock(&channel->rd_spinlock); 206 207 wake_up_interruptible(&channel->rd_wait); 208 if (!channel->rd_synchronous) 209 queue_delayed_work( 210 xillybus_wq, 211 &channel->rd_workitem, 212 XILLY_RX_TIMEOUT); 213 } 214 215 break; 216 case XILLYMSG_OPCODE_NONEMPTY: 217 if ((msg_channel > ep->num_channels) || 218 (msg_channel == 0) || (!msg_dir) || 219 !ep->channels[msg_channel]->wr_supports_nonempty) { 220 malformed_message(ep, &buf[i]); 221 break; 222 } 223 224 channel = ep->channels[msg_channel]; 225 226 if (msg_bufno >= channel->num_wr_buffers) { 227 malformed_message(ep, &buf[i]); 228 break; 229 } 230 spin_lock(&channel->wr_spinlock); 231 if (msg_bufno == channel->wr_host_buf_idx) 232 channel->wr_ready = 1; 233 spin_unlock(&channel->wr_spinlock); 234 235 wake_up_interruptible(&channel->wr_ready_wait); 236 237 break; 238 case XILLYMSG_OPCODE_QUIESCEACK: 239 ep->idtlen = msg_data; 240 wake_up_interruptible(&ep->ep_wait); 241 242 break; 243 case XILLYMSG_OPCODE_FIFOEOF: 244 if ((msg_channel > ep->num_channels) || 245 (msg_channel == 0) || (!msg_dir) || 246 !ep->channels[msg_channel]->num_wr_buffers) { 247 malformed_message(ep, &buf[i]); 248 break; 249 } 250 channel = ep->channels[msg_channel]; 251 spin_lock(&channel->wr_spinlock); 252 channel->wr_eof = msg_bufno; 253 channel->wr_sleepy = 0; 254 255 channel->wr_hangup = channel->wr_empty && 256 (channel->wr_host_buf_idx == msg_bufno); 257 258 spin_unlock(&channel->wr_spinlock); 259 260 wake_up_interruptible(&channel->wr_wait); 261 262 break; 263 case XILLYMSG_OPCODE_FATAL_ERROR: 264 ep->fatal_error = 1; 265 wake_up_interruptible(&ep->ep_wait); /* For select() */ 266 dev_err(ep->dev, 267 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); 268 break; 269 default: 270 malformed_message(ep, &buf[i]); 271 break; 272 } 273 } 274 275 dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr, 276 ep->msg_buf_size, DMA_FROM_DEVICE); 277 278 ep->msg_counter = (ep->msg_counter + 1) & 0xf; 279 ep->failed_messages = 0; 280 iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ 281 282 return IRQ_HANDLED; 283 } 284 EXPORT_SYMBOL(xillybus_isr); 285 286 /* 287 * A few trivial memory management functions. 288 * NOTE: These functions are used only on probe and remove, and therefore 289 * no locks are applied! 290 */ 291 292 static void xillybus_autoflush(struct work_struct *work); 293 294 struct xilly_alloc_state { 295 void *salami; 296 int left_of_salami; 297 int nbuffer; 298 enum dma_data_direction direction; 299 u32 regdirection; 300 }; 301 302 static void xilly_unmap(void *ptr) 303 { 304 struct xilly_mapping *data = ptr; 305 306 dma_unmap_single(data->device, data->dma_addr, 307 data->size, data->direction); 308 309 kfree(ptr); 310 } 311 312 static int xilly_map_single(struct xilly_endpoint *ep, 313 void *ptr, 314 size_t size, 315 int direction, 316 dma_addr_t *ret_dma_handle 317 ) 318 { 319 dma_addr_t addr; 320 struct xilly_mapping *this; 321 322 this = kzalloc(sizeof(*this), GFP_KERNEL); 323 if (!this) 324 return -ENOMEM; 325 326 addr = dma_map_single(ep->dev, ptr, size, direction); 327 328 if (dma_mapping_error(ep->dev, addr)) { 329 kfree(this); 330 return -ENODEV; 331 } 332 333 this->device = ep->dev; 334 this->dma_addr = addr; 335 this->size = size; 336 this->direction = direction; 337 338 *ret_dma_handle = addr; 339 340 return devm_add_action_or_reset(ep->dev, xilly_unmap, this); 341 } 342 343 static int xilly_get_dma_buffers(struct xilly_endpoint *ep, 344 struct xilly_alloc_state *s, 345 struct xilly_buffer **buffers, 346 int bufnum, int bytebufsize) 347 { 348 int i, rc; 349 dma_addr_t dma_addr; 350 struct device *dev = ep->dev; 351 struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ 352 353 if (buffers) { /* Not the message buffer */ 354 this_buffer = devm_kcalloc(dev, bufnum, 355 sizeof(struct xilly_buffer), 356 GFP_KERNEL); 357 if (!this_buffer) 358 return -ENOMEM; 359 } 360 361 for (i = 0; i < bufnum; i++) { 362 /* 363 * Buffers are expected in descending size order, so there 364 * is either enough space for this buffer or none at all. 365 */ 366 367 if ((s->left_of_salami < bytebufsize) && 368 (s->left_of_salami > 0)) { 369 dev_err(ep->dev, 370 "Corrupt buffer allocation in IDT. Aborting.\n"); 371 return -ENODEV; 372 } 373 374 if (s->left_of_salami == 0) { 375 int allocorder, allocsize; 376 377 allocsize = PAGE_SIZE; 378 allocorder = 0; 379 while (bytebufsize > allocsize) { 380 allocsize *= 2; 381 allocorder++; 382 } 383 384 s->salami = (void *) devm_get_free_pages( 385 dev, 386 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, 387 allocorder); 388 if (!s->salami) 389 return -ENOMEM; 390 391 s->left_of_salami = allocsize; 392 } 393 394 rc = xilly_map_single(ep, s->salami, 395 bytebufsize, s->direction, 396 &dma_addr); 397 if (rc) 398 return rc; 399 400 iowrite32((u32) (dma_addr & 0xffffffff), 401 ep->registers + fpga_dma_bufaddr_lowaddr_reg); 402 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), 403 ep->registers + fpga_dma_bufaddr_highaddr_reg); 404 405 if (buffers) { /* Not the message buffer */ 406 this_buffer->addr = s->salami; 407 this_buffer->dma_addr = dma_addr; 408 buffers[i] = this_buffer++; 409 410 iowrite32(s->regdirection | s->nbuffer++, 411 ep->registers + fpga_dma_bufno_reg); 412 } else { 413 ep->msgbuf_addr = s->salami; 414 ep->msgbuf_dma_addr = dma_addr; 415 ep->msg_buf_size = bytebufsize; 416 417 iowrite32(s->regdirection, 418 ep->registers + fpga_dma_bufno_reg); 419 } 420 421 s->left_of_salami -= bytebufsize; 422 s->salami += bytebufsize; 423 } 424 return 0; 425 } 426 427 static int xilly_setupchannels(struct xilly_endpoint *ep, 428 unsigned char *chandesc, 429 int entries) 430 { 431 struct device *dev = ep->dev; 432 int i, entry, rc; 433 struct xilly_channel *channel; 434 int channelnum, bufnum, bufsize, format, is_writebuf; 435 int bytebufsize; 436 int synchronous, allowpartial, exclusive_open, seekable; 437 int supports_nonempty; 438 int msg_buf_done = 0; 439 440 struct xilly_alloc_state rd_alloc = { 441 .salami = NULL, 442 .left_of_salami = 0, 443 .nbuffer = 1, 444 .direction = DMA_TO_DEVICE, 445 .regdirection = 0, 446 }; 447 448 struct xilly_alloc_state wr_alloc = { 449 .salami = NULL, 450 .left_of_salami = 0, 451 .nbuffer = 1, 452 .direction = DMA_FROM_DEVICE, 453 .regdirection = 0x80000000, 454 }; 455 456 channel = devm_kcalloc(dev, ep->num_channels, 457 sizeof(struct xilly_channel), GFP_KERNEL); 458 if (!channel) 459 return -ENOMEM; 460 461 ep->channels = devm_kcalloc(dev, ep->num_channels + 1, 462 sizeof(struct xilly_channel *), 463 GFP_KERNEL); 464 if (!ep->channels) 465 return -ENOMEM; 466 467 ep->channels[0] = NULL; /* Channel 0 is message buf. */ 468 469 /* Initialize all channels with defaults */ 470 471 for (i = 1; i <= ep->num_channels; i++) { 472 channel->wr_buffers = NULL; 473 channel->rd_buffers = NULL; 474 channel->num_wr_buffers = 0; 475 channel->num_rd_buffers = 0; 476 channel->wr_fpga_buf_idx = -1; 477 channel->wr_host_buf_idx = 0; 478 channel->wr_host_buf_pos = 0; 479 channel->wr_empty = 1; 480 channel->wr_ready = 0; 481 channel->wr_sleepy = 1; 482 channel->rd_fpga_buf_idx = 0; 483 channel->rd_host_buf_idx = 0; 484 channel->rd_host_buf_pos = 0; 485 channel->rd_full = 0; 486 channel->wr_ref_count = 0; 487 channel->rd_ref_count = 0; 488 489 spin_lock_init(&channel->wr_spinlock); 490 spin_lock_init(&channel->rd_spinlock); 491 mutex_init(&channel->wr_mutex); 492 mutex_init(&channel->rd_mutex); 493 init_waitqueue_head(&channel->rd_wait); 494 init_waitqueue_head(&channel->wr_wait); 495 init_waitqueue_head(&channel->wr_ready_wait); 496 497 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); 498 499 channel->endpoint = ep; 500 channel->chan_num = i; 501 502 channel->log2_element_size = 0; 503 504 ep->channels[i] = channel++; 505 } 506 507 for (entry = 0; entry < entries; entry++, chandesc += 4) { 508 struct xilly_buffer **buffers = NULL; 509 510 is_writebuf = chandesc[0] & 0x01; 511 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); 512 format = (chandesc[1] >> 4) & 0x03; 513 allowpartial = (chandesc[1] >> 6) & 0x01; 514 synchronous = (chandesc[1] >> 7) & 0x01; 515 bufsize = 1 << (chandesc[2] & 0x1f); 516 bufnum = 1 << (chandesc[3] & 0x0f); 517 exclusive_open = (chandesc[2] >> 7) & 0x01; 518 seekable = (chandesc[2] >> 6) & 0x01; 519 supports_nonempty = (chandesc[2] >> 5) & 0x01; 520 521 if ((channelnum > ep->num_channels) || 522 ((channelnum == 0) && !is_writebuf)) { 523 dev_err(ep->dev, 524 "IDT requests channel out of range. Aborting.\n"); 525 return -ENODEV; 526 } 527 528 channel = ep->channels[channelnum]; /* NULL for msg channel */ 529 530 if (!is_writebuf || channelnum > 0) { 531 channel->log2_element_size = ((format > 2) ? 532 2 : format); 533 534 bytebufsize = bufsize * 535 (1 << channel->log2_element_size); 536 537 buffers = devm_kcalloc(dev, bufnum, 538 sizeof(struct xilly_buffer *), 539 GFP_KERNEL); 540 if (!buffers) 541 return -ENOMEM; 542 } else { 543 bytebufsize = bufsize << 2; 544 } 545 546 if (!is_writebuf) { 547 channel->num_rd_buffers = bufnum; 548 channel->rd_buf_size = bytebufsize; 549 channel->rd_allow_partial = allowpartial; 550 channel->rd_synchronous = synchronous; 551 channel->rd_exclusive_open = exclusive_open; 552 channel->seekable = seekable; 553 554 channel->rd_buffers = buffers; 555 rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, 556 bufnum, bytebufsize); 557 } else if (channelnum > 0) { 558 channel->num_wr_buffers = bufnum; 559 channel->wr_buf_size = bytebufsize; 560 561 channel->seekable = seekable; 562 channel->wr_supports_nonempty = supports_nonempty; 563 564 channel->wr_allow_partial = allowpartial; 565 channel->wr_synchronous = synchronous; 566 channel->wr_exclusive_open = exclusive_open; 567 568 channel->wr_buffers = buffers; 569 rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, 570 bufnum, bytebufsize); 571 } else { 572 rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, 573 bufnum, bytebufsize); 574 msg_buf_done++; 575 } 576 577 if (rc) 578 return -ENOMEM; 579 } 580 581 if (!msg_buf_done) { 582 dev_err(ep->dev, 583 "Corrupt IDT: No message buffer. Aborting.\n"); 584 return -ENODEV; 585 } 586 return 0; 587 } 588 589 static int xilly_scan_idt(struct xilly_endpoint *endpoint, 590 struct xilly_idt_handle *idt_handle) 591 { 592 int count = 0; 593 unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; 594 unsigned char *end_of_idt = idt + endpoint->idtlen - 4; 595 unsigned char *scan; 596 int len; 597 598 scan = idt + 1; 599 idt_handle->names = scan; 600 601 while ((scan <= end_of_idt) && *scan) { 602 while ((scan <= end_of_idt) && *scan++) 603 /* Do nothing, just scan thru string */; 604 count++; 605 } 606 607 idt_handle->names_len = scan - idt_handle->names; 608 609 scan++; 610 611 if (scan > end_of_idt) { 612 dev_err(endpoint->dev, 613 "IDT device name list overflow. Aborting.\n"); 614 return -ENODEV; 615 } 616 idt_handle->chandesc = scan; 617 618 len = endpoint->idtlen - (3 + ((int) (scan - idt))); 619 620 if (len & 0x03) { 621 dev_err(endpoint->dev, 622 "Corrupt IDT device name list. Aborting.\n"); 623 return -ENODEV; 624 } 625 626 idt_handle->entries = len >> 2; 627 endpoint->num_channels = count; 628 629 return 0; 630 } 631 632 static int xilly_obtain_idt(struct xilly_endpoint *endpoint) 633 { 634 struct xilly_channel *channel; 635 unsigned char *version; 636 long t; 637 638 channel = endpoint->channels[1]; /* This should be generated ad-hoc */ 639 640 channel->wr_sleepy = 1; 641 642 iowrite32(1 | 643 (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ 644 endpoint->registers + fpga_buf_ctrl_reg); 645 646 t = wait_event_interruptible_timeout(channel->wr_wait, 647 (!channel->wr_sleepy), 648 XILLY_TIMEOUT); 649 650 if (t <= 0) { 651 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); 652 653 if (endpoint->fatal_error) 654 return -EIO; 655 656 return -ENODEV; 657 } 658 659 dma_sync_single_for_cpu(channel->endpoint->dev, 660 channel->wr_buffers[0]->dma_addr, 661 channel->wr_buf_size, 662 DMA_FROM_DEVICE); 663 664 if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { 665 dev_err(endpoint->dev, 666 "IDT length mismatch (%d != %d). Aborting.\n", 667 channel->wr_buffers[0]->end_offset, endpoint->idtlen); 668 return -ENODEV; 669 } 670 671 if (crc32_le(~0, channel->wr_buffers[0]->addr, 672 endpoint->idtlen+1) != 0) { 673 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); 674 return -ENODEV; 675 } 676 677 version = channel->wr_buffers[0]->addr; 678 679 /* Check version number. Reject anything above 0x82. */ 680 if (*version > 0x82) { 681 dev_err(endpoint->dev, 682 "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n", 683 *version); 684 return -ENODEV; 685 } 686 687 return 0; 688 } 689 690 static ssize_t xillybus_read(struct file *filp, char __user *userbuf, 691 size_t count, loff_t *f_pos) 692 { 693 ssize_t rc; 694 unsigned long flags; 695 int bytes_done = 0; 696 int no_time_left = 0; 697 long deadline, left_to_sleep; 698 struct xilly_channel *channel = filp->private_data; 699 700 int empty, reached_eof, exhausted, ready; 701 /* Initializations are there only to silence warnings */ 702 703 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; 704 int waiting_bufidx; 705 706 if (channel->endpoint->fatal_error) 707 return -EIO; 708 709 deadline = jiffies + 1 + XILLY_RX_TIMEOUT; 710 711 rc = mutex_lock_interruptible(&channel->wr_mutex); 712 if (rc) 713 return rc; 714 715 while (1) { /* Note that we may drop mutex within this loop */ 716 int bytes_to_do = count - bytes_done; 717 718 spin_lock_irqsave(&channel->wr_spinlock, flags); 719 720 empty = channel->wr_empty; 721 ready = !empty || channel->wr_ready; 722 723 if (!empty) { 724 bufidx = channel->wr_host_buf_idx; 725 bufpos = channel->wr_host_buf_pos; 726 howmany = ((channel->wr_buffers[bufidx]->end_offset 727 + 1) << channel->log2_element_size) 728 - bufpos; 729 730 /* Update wr_host_* to its post-operation state */ 731 if (howmany > bytes_to_do) { 732 bufferdone = 0; 733 734 howmany = bytes_to_do; 735 channel->wr_host_buf_pos += howmany; 736 } else { 737 bufferdone = 1; 738 739 channel->wr_host_buf_pos = 0; 740 741 if (bufidx == channel->wr_fpga_buf_idx) { 742 channel->wr_empty = 1; 743 channel->wr_sleepy = 1; 744 channel->wr_ready = 0; 745 } 746 747 if (bufidx >= (channel->num_wr_buffers - 1)) 748 channel->wr_host_buf_idx = 0; 749 else 750 channel->wr_host_buf_idx++; 751 } 752 } 753 754 /* 755 * Marking our situation after the possible changes above, 756 * for use after releasing the spinlock. 757 * 758 * empty = empty before change 759 * exhasted = empty after possible change 760 */ 761 762 reached_eof = channel->wr_empty && 763 (channel->wr_host_buf_idx == channel->wr_eof); 764 channel->wr_hangup = reached_eof; 765 exhausted = channel->wr_empty; 766 waiting_bufidx = channel->wr_host_buf_idx; 767 768 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 769 770 if (!empty) { /* Go on, now without the spinlock */ 771 772 if (bufpos == 0) /* Position zero means it's virgin */ 773 dma_sync_single_for_cpu(channel->endpoint->dev, 774 channel->wr_buffers[bufidx]->dma_addr, 775 channel->wr_buf_size, 776 DMA_FROM_DEVICE); 777 778 if (copy_to_user( 779 userbuf, 780 channel->wr_buffers[bufidx]->addr 781 + bufpos, howmany)) 782 rc = -EFAULT; 783 784 userbuf += howmany; 785 bytes_done += howmany; 786 787 if (bufferdone) { 788 dma_sync_single_for_device(channel->endpoint->dev, 789 channel->wr_buffers[bufidx]->dma_addr, 790 channel->wr_buf_size, 791 DMA_FROM_DEVICE); 792 793 /* 794 * Tell FPGA the buffer is done with. It's an 795 * atomic operation to the FPGA, so what 796 * happens with other channels doesn't matter, 797 * and the certain channel is protected with 798 * the channel-specific mutex. 799 */ 800 801 iowrite32(1 | (channel->chan_num << 1) | 802 (bufidx << 12), 803 channel->endpoint->registers + 804 fpga_buf_ctrl_reg); 805 } 806 807 if (rc) { 808 mutex_unlock(&channel->wr_mutex); 809 return rc; 810 } 811 } 812 813 /* This includes a zero-count return = EOF */ 814 if ((bytes_done >= count) || reached_eof) 815 break; 816 817 if (!exhausted) 818 continue; /* More in RAM buffer(s)? Just go on. */ 819 820 if ((bytes_done > 0) && 821 (no_time_left || 822 (channel->wr_synchronous && channel->wr_allow_partial))) 823 break; 824 825 /* 826 * Nonblocking read: The "ready" flag tells us that the FPGA 827 * has data to send. In non-blocking mode, if it isn't on, 828 * just return. But if there is, we jump directly to the point 829 * where we ask for the FPGA to send all it has, and wait 830 * until that data arrives. So in a sense, we *do* block in 831 * nonblocking mode, but only for a very short time. 832 */ 833 834 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { 835 if (bytes_done > 0) 836 break; 837 838 if (ready) 839 goto desperate; 840 841 rc = -EAGAIN; 842 break; 843 } 844 845 if (!no_time_left || (bytes_done > 0)) { 846 /* 847 * Note that in case of an element-misaligned read 848 * request, offsetlimit will include the last element, 849 * which will be partially read from. 850 */ 851 int offsetlimit = ((count - bytes_done) - 1) >> 852 channel->log2_element_size; 853 int buf_elements = channel->wr_buf_size >> 854 channel->log2_element_size; 855 856 /* 857 * In synchronous mode, always send an offset limit. 858 * Just don't send a value too big. 859 */ 860 861 if (channel->wr_synchronous) { 862 /* Don't request more than one buffer */ 863 if (channel->wr_allow_partial && 864 (offsetlimit >= buf_elements)) 865 offsetlimit = buf_elements - 1; 866 867 /* Don't request more than all buffers */ 868 if (!channel->wr_allow_partial && 869 (offsetlimit >= 870 (buf_elements * channel->num_wr_buffers))) 871 offsetlimit = buf_elements * 872 channel->num_wr_buffers - 1; 873 } 874 875 /* 876 * In asynchronous mode, force early flush of a buffer 877 * only if that will allow returning a full count. The 878 * "offsetlimit < ( ... )" rather than "<=" excludes 879 * requesting a full buffer, which would obviously 880 * cause a buffer transmission anyhow 881 */ 882 883 if (channel->wr_synchronous || 884 (offsetlimit < (buf_elements - 1))) { 885 mutex_lock(&channel->endpoint->register_mutex); 886 887 iowrite32(offsetlimit, 888 channel->endpoint->registers + 889 fpga_buf_offset_reg); 890 891 iowrite32(1 | (channel->chan_num << 1) | 892 (2 << 24) | /* 2 = offset limit */ 893 (waiting_bufidx << 12), 894 channel->endpoint->registers + 895 fpga_buf_ctrl_reg); 896 897 mutex_unlock(&channel->endpoint-> 898 register_mutex); 899 } 900 } 901 902 /* 903 * If partial completion is disallowed, there is no point in 904 * timeout sleeping. Neither if no_time_left is set and 905 * there's no data. 906 */ 907 908 if (!channel->wr_allow_partial || 909 (no_time_left && (bytes_done == 0))) { 910 /* 911 * This do-loop will run more than once if another 912 * thread reasserted wr_sleepy before we got the mutex 913 * back, so we try again. 914 */ 915 916 do { 917 mutex_unlock(&channel->wr_mutex); 918 919 if (wait_event_interruptible( 920 channel->wr_wait, 921 (!channel->wr_sleepy))) 922 goto interrupted; 923 924 if (mutex_lock_interruptible( 925 &channel->wr_mutex)) 926 goto interrupted; 927 } while (channel->wr_sleepy); 928 929 continue; 930 931 interrupted: /* Mutex is not held if got here */ 932 if (channel->endpoint->fatal_error) 933 return -EIO; 934 if (bytes_done) 935 return bytes_done; 936 if (filp->f_flags & O_NONBLOCK) 937 return -EAGAIN; /* Don't admit snoozing */ 938 return -EINTR; 939 } 940 941 left_to_sleep = deadline - ((long) jiffies); 942 943 /* 944 * If our time is out, skip the waiting. We may miss wr_sleepy 945 * being deasserted but hey, almost missing the train is like 946 * missing it. 947 */ 948 949 if (left_to_sleep > 0) { 950 left_to_sleep = 951 wait_event_interruptible_timeout( 952 channel->wr_wait, 953 (!channel->wr_sleepy), 954 left_to_sleep); 955 956 if (left_to_sleep > 0) /* wr_sleepy deasserted */ 957 continue; 958 959 if (left_to_sleep < 0) { /* Interrupt */ 960 mutex_unlock(&channel->wr_mutex); 961 if (channel->endpoint->fatal_error) 962 return -EIO; 963 if (bytes_done) 964 return bytes_done; 965 return -EINTR; 966 } 967 } 968 969 desperate: 970 no_time_left = 1; /* We're out of sleeping time. Desperate! */ 971 972 if (bytes_done == 0) { 973 /* 974 * Reaching here means that we allow partial return, 975 * that we've run out of time, and that we have 976 * nothing to return. 977 * So tell the FPGA to send anything it has or gets. 978 */ 979 980 iowrite32(1 | (channel->chan_num << 1) | 981 (3 << 24) | /* Opcode 3, flush it all! */ 982 (waiting_bufidx << 12), 983 channel->endpoint->registers + 984 fpga_buf_ctrl_reg); 985 } 986 987 /* 988 * Reaching here means that we *do* have data in the buffer, 989 * but the "partial" flag disallows returning less than 990 * required. And we don't have as much. So loop again, 991 * which is likely to end up blocking indefinitely until 992 * enough data has arrived. 993 */ 994 } 995 996 mutex_unlock(&channel->wr_mutex); 997 998 if (channel->endpoint->fatal_error) 999 return -EIO; 1000 1001 if (rc) 1002 return rc; 1003 1004 return bytes_done; 1005 } 1006 1007 /* 1008 * The timeout argument takes values as follows: 1009 * >0 : Flush with timeout 1010 * ==0 : Flush, and wait idefinitely for the flush to complete 1011 * <0 : Autoflush: Flush only if there's a single buffer occupied 1012 */ 1013 1014 static int xillybus_myflush(struct xilly_channel *channel, long timeout) 1015 { 1016 int rc; 1017 unsigned long flags; 1018 1019 int end_offset_plus1; 1020 int bufidx, bufidx_minus1; 1021 int i; 1022 int empty; 1023 int new_rd_host_buf_pos; 1024 1025 if (channel->endpoint->fatal_error) 1026 return -EIO; 1027 rc = mutex_lock_interruptible(&channel->rd_mutex); 1028 if (rc) 1029 return rc; 1030 1031 /* 1032 * Don't flush a closed channel. This can happen when the work queued 1033 * autoflush thread fires off after the file has closed. This is not 1034 * an error, just something to dismiss. 1035 */ 1036 1037 if (!channel->rd_ref_count) 1038 goto done; 1039 1040 bufidx = channel->rd_host_buf_idx; 1041 1042 bufidx_minus1 = (bufidx == 0) ? 1043 channel->num_rd_buffers - 1 : 1044 bufidx - 1; 1045 1046 end_offset_plus1 = channel->rd_host_buf_pos >> 1047 channel->log2_element_size; 1048 1049 new_rd_host_buf_pos = channel->rd_host_buf_pos - 1050 (end_offset_plus1 << channel->log2_element_size); 1051 1052 /* Submit the current buffer if it's nonempty */ 1053 if (end_offset_plus1) { 1054 unsigned char *tail = channel->rd_buffers[bufidx]->addr + 1055 (end_offset_plus1 << channel->log2_element_size); 1056 1057 /* Copy unflushed data, so we can put it in next buffer */ 1058 for (i = 0; i < new_rd_host_buf_pos; i++) 1059 channel->rd_leftovers[i] = *tail++; 1060 1061 spin_lock_irqsave(&channel->rd_spinlock, flags); 1062 1063 /* Autoflush only if a single buffer is occupied */ 1064 1065 if ((timeout < 0) && 1066 (channel->rd_full || 1067 (bufidx_minus1 != channel->rd_fpga_buf_idx))) { 1068 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1069 /* 1070 * A new work item may be queued by the ISR exactly 1071 * now, since the execution of a work item allows the 1072 * queuing of a new one while it's running. 1073 */ 1074 goto done; 1075 } 1076 1077 /* The 4th element is never needed for data, so it's a flag */ 1078 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); 1079 1080 /* Set up rd_full to reflect a certain moment's state */ 1081 1082 if (bufidx == channel->rd_fpga_buf_idx) 1083 channel->rd_full = 1; 1084 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1085 1086 if (bufidx >= (channel->num_rd_buffers - 1)) 1087 channel->rd_host_buf_idx = 0; 1088 else 1089 channel->rd_host_buf_idx++; 1090 1091 dma_sync_single_for_device(channel->endpoint->dev, 1092 channel->rd_buffers[bufidx]->dma_addr, 1093 channel->rd_buf_size, 1094 DMA_TO_DEVICE); 1095 1096 mutex_lock(&channel->endpoint->register_mutex); 1097 1098 iowrite32(end_offset_plus1 - 1, 1099 channel->endpoint->registers + fpga_buf_offset_reg); 1100 1101 iowrite32((channel->chan_num << 1) | /* Channel ID */ 1102 (2 << 24) | /* Opcode 2, submit buffer */ 1103 (bufidx << 12), 1104 channel->endpoint->registers + fpga_buf_ctrl_reg); 1105 1106 mutex_unlock(&channel->endpoint->register_mutex); 1107 } else if (bufidx == 0) { 1108 bufidx = channel->num_rd_buffers - 1; 1109 } else { 1110 bufidx--; 1111 } 1112 1113 channel->rd_host_buf_pos = new_rd_host_buf_pos; 1114 1115 if (timeout < 0) 1116 goto done; /* Autoflush */ 1117 1118 /* 1119 * bufidx is now the last buffer written to (or equal to 1120 * rd_fpga_buf_idx if buffer was never written to), and 1121 * channel->rd_host_buf_idx the one after it. 1122 * 1123 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. 1124 */ 1125 1126 while (1) { /* Loop waiting for draining of buffers */ 1127 spin_lock_irqsave(&channel->rd_spinlock, flags); 1128 1129 if (bufidx != channel->rd_fpga_buf_idx) 1130 channel->rd_full = 1; /* 1131 * Not really full, 1132 * but needs waiting. 1133 */ 1134 1135 empty = !channel->rd_full; 1136 1137 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1138 1139 if (empty) 1140 break; 1141 1142 /* 1143 * Indefinite sleep with mutex taken. With data waiting for 1144 * flushing user should not be surprised if open() for write 1145 * sleeps. 1146 */ 1147 if (timeout == 0) 1148 wait_event_interruptible(channel->rd_wait, 1149 (!channel->rd_full)); 1150 1151 else if (wait_event_interruptible_timeout( 1152 channel->rd_wait, 1153 (!channel->rd_full), 1154 timeout) == 0) { 1155 dev_warn(channel->endpoint->dev, 1156 "Timed out while flushing. Output data may be lost.\n"); 1157 1158 rc = -ETIMEDOUT; 1159 break; 1160 } 1161 1162 if (channel->rd_full) { 1163 rc = -EINTR; 1164 break; 1165 } 1166 } 1167 1168 done: 1169 mutex_unlock(&channel->rd_mutex); 1170 1171 if (channel->endpoint->fatal_error) 1172 return -EIO; 1173 1174 return rc; 1175 } 1176 1177 static int xillybus_flush(struct file *filp, fl_owner_t id) 1178 { 1179 if (!(filp->f_mode & FMODE_WRITE)) 1180 return 0; 1181 1182 return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */ 1183 } 1184 1185 static void xillybus_autoflush(struct work_struct *work) 1186 { 1187 struct delayed_work *workitem = to_delayed_work(work); 1188 struct xilly_channel *channel = container_of( 1189 workitem, struct xilly_channel, rd_workitem); 1190 int rc; 1191 1192 rc = xillybus_myflush(channel, -1); 1193 if (rc == -EINTR) 1194 dev_warn(channel->endpoint->dev, 1195 "Autoflush failed because work queue thread got a signal.\n"); 1196 else if (rc) 1197 dev_err(channel->endpoint->dev, 1198 "Autoflush failed under weird circumstances.\n"); 1199 } 1200 1201 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf, 1202 size_t count, loff_t *f_pos) 1203 { 1204 ssize_t rc; 1205 unsigned long flags; 1206 int bytes_done = 0; 1207 struct xilly_channel *channel = filp->private_data; 1208 1209 int full, exhausted; 1210 /* Initializations are there only to silence warnings */ 1211 1212 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; 1213 int end_offset_plus1 = 0; 1214 1215 if (channel->endpoint->fatal_error) 1216 return -EIO; 1217 1218 rc = mutex_lock_interruptible(&channel->rd_mutex); 1219 if (rc) 1220 return rc; 1221 1222 while (1) { 1223 int bytes_to_do = count - bytes_done; 1224 1225 spin_lock_irqsave(&channel->rd_spinlock, flags); 1226 1227 full = channel->rd_full; 1228 1229 if (!full) { 1230 bufidx = channel->rd_host_buf_idx; 1231 bufpos = channel->rd_host_buf_pos; 1232 howmany = channel->rd_buf_size - bufpos; 1233 1234 /* 1235 * Update rd_host_* to its state after this operation. 1236 * count=0 means committing the buffer immediately, 1237 * which is like flushing, but not necessarily block. 1238 */ 1239 1240 if ((howmany > bytes_to_do) && 1241 (count || 1242 ((bufpos >> channel->log2_element_size) == 0))) { 1243 bufferdone = 0; 1244 1245 howmany = bytes_to_do; 1246 channel->rd_host_buf_pos += howmany; 1247 } else { 1248 bufferdone = 1; 1249 1250 if (count) { 1251 end_offset_plus1 = 1252 channel->rd_buf_size >> 1253 channel->log2_element_size; 1254 channel->rd_host_buf_pos = 0; 1255 } else { 1256 unsigned char *tail; 1257 int i; 1258 1259 howmany = 0; 1260 1261 end_offset_plus1 = bufpos >> 1262 channel->log2_element_size; 1263 1264 channel->rd_host_buf_pos -= 1265 end_offset_plus1 << 1266 channel->log2_element_size; 1267 1268 tail = channel-> 1269 rd_buffers[bufidx]->addr + 1270 (end_offset_plus1 << 1271 channel->log2_element_size); 1272 1273 for (i = 0; 1274 i < channel->rd_host_buf_pos; 1275 i++) 1276 channel->rd_leftovers[i] = 1277 *tail++; 1278 } 1279 1280 if (bufidx == channel->rd_fpga_buf_idx) 1281 channel->rd_full = 1; 1282 1283 if (bufidx >= (channel->num_rd_buffers - 1)) 1284 channel->rd_host_buf_idx = 0; 1285 else 1286 channel->rd_host_buf_idx++; 1287 } 1288 } 1289 1290 /* 1291 * Marking our situation after the possible changes above, 1292 * for use after releasing the spinlock. 1293 * 1294 * full = full before change 1295 * exhasted = full after possible change 1296 */ 1297 1298 exhausted = channel->rd_full; 1299 1300 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1301 1302 if (!full) { /* Go on, now without the spinlock */ 1303 unsigned char *head = 1304 channel->rd_buffers[bufidx]->addr; 1305 int i; 1306 1307 if ((bufpos == 0) || /* Zero means it's virgin */ 1308 (channel->rd_leftovers[3] != 0)) { 1309 dma_sync_single_for_cpu(channel->endpoint->dev, 1310 channel->rd_buffers[bufidx]->dma_addr, 1311 channel->rd_buf_size, 1312 DMA_TO_DEVICE); 1313 1314 /* Virgin, but leftovers are due */ 1315 for (i = 0; i < bufpos; i++) 1316 *head++ = channel->rd_leftovers[i]; 1317 1318 channel->rd_leftovers[3] = 0; /* Clear flag */ 1319 } 1320 1321 if (copy_from_user( 1322 channel->rd_buffers[bufidx]->addr + bufpos, 1323 userbuf, howmany)) 1324 rc = -EFAULT; 1325 1326 userbuf += howmany; 1327 bytes_done += howmany; 1328 1329 if (bufferdone) { 1330 dma_sync_single_for_device(channel->endpoint->dev, 1331 channel->rd_buffers[bufidx]->dma_addr, 1332 channel->rd_buf_size, 1333 DMA_TO_DEVICE); 1334 1335 mutex_lock(&channel->endpoint->register_mutex); 1336 1337 iowrite32(end_offset_plus1 - 1, 1338 channel->endpoint->registers + 1339 fpga_buf_offset_reg); 1340 1341 iowrite32((channel->chan_num << 1) | 1342 (2 << 24) | /* 2 = submit buffer */ 1343 (bufidx << 12), 1344 channel->endpoint->registers + 1345 fpga_buf_ctrl_reg); 1346 1347 mutex_unlock(&channel->endpoint-> 1348 register_mutex); 1349 1350 channel->rd_leftovers[3] = 1351 (channel->rd_host_buf_pos != 0); 1352 } 1353 1354 if (rc) { 1355 mutex_unlock(&channel->rd_mutex); 1356 1357 if (channel->endpoint->fatal_error) 1358 return -EIO; 1359 1360 if (!channel->rd_synchronous) 1361 queue_delayed_work( 1362 xillybus_wq, 1363 &channel->rd_workitem, 1364 XILLY_RX_TIMEOUT); 1365 1366 return rc; 1367 } 1368 } 1369 1370 if (bytes_done >= count) 1371 break; 1372 1373 if (!exhausted) 1374 continue; /* If there's more space, just go on */ 1375 1376 if ((bytes_done > 0) && channel->rd_allow_partial) 1377 break; 1378 1379 /* 1380 * Indefinite sleep with mutex taken. With data waiting for 1381 * flushing, user should not be surprised if open() for write 1382 * sleeps. 1383 */ 1384 1385 if (filp->f_flags & O_NONBLOCK) { 1386 rc = -EAGAIN; 1387 break; 1388 } 1389 1390 if (wait_event_interruptible(channel->rd_wait, 1391 (!channel->rd_full))) { 1392 mutex_unlock(&channel->rd_mutex); 1393 1394 if (channel->endpoint->fatal_error) 1395 return -EIO; 1396 1397 if (bytes_done) 1398 return bytes_done; 1399 return -EINTR; 1400 } 1401 } 1402 1403 mutex_unlock(&channel->rd_mutex); 1404 1405 if (!channel->rd_synchronous) 1406 queue_delayed_work(xillybus_wq, 1407 &channel->rd_workitem, 1408 XILLY_RX_TIMEOUT); 1409 1410 if (channel->endpoint->fatal_error) 1411 return -EIO; 1412 1413 if (rc) 1414 return rc; 1415 1416 if ((channel->rd_synchronous) && (bytes_done > 0)) { 1417 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */ 1418 1419 if (rc && (rc != -EINTR)) 1420 return rc; 1421 } 1422 1423 return bytes_done; 1424 } 1425 1426 static int xillybus_open(struct inode *inode, struct file *filp) 1427 { 1428 int rc; 1429 unsigned long flags; 1430 struct xilly_endpoint *endpoint; 1431 struct xilly_channel *channel; 1432 int index; 1433 1434 rc = xillybus_find_inode(inode, (void **)&endpoint, &index); 1435 if (rc) 1436 return rc; 1437 1438 if (endpoint->fatal_error) 1439 return -EIO; 1440 1441 channel = endpoint->channels[1 + index]; 1442 filp->private_data = channel; 1443 1444 /* 1445 * It gets complicated because: 1446 * 1. We don't want to take a mutex we don't have to 1447 * 2. We don't want to open one direction if the other will fail. 1448 */ 1449 1450 if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers)) 1451 return -ENODEV; 1452 1453 if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers)) 1454 return -ENODEV; 1455 1456 if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) && 1457 (channel->wr_synchronous || !channel->wr_allow_partial || 1458 !channel->wr_supports_nonempty)) { 1459 dev_err(endpoint->dev, 1460 "open() failed: O_NONBLOCK not allowed for read on this device\n"); 1461 return -ENODEV; 1462 } 1463 1464 if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) && 1465 (channel->rd_synchronous || !channel->rd_allow_partial)) { 1466 dev_err(endpoint->dev, 1467 "open() failed: O_NONBLOCK not allowed for write on this device\n"); 1468 return -ENODEV; 1469 } 1470 1471 /* 1472 * Note: open() may block on getting mutexes despite O_NONBLOCK. 1473 * This shouldn't occur normally, since multiple open of the same 1474 * file descriptor is almost always prohibited anyhow 1475 * (*_exclusive_open is normally set in real-life systems). 1476 */ 1477 1478 if (filp->f_mode & FMODE_READ) { 1479 rc = mutex_lock_interruptible(&channel->wr_mutex); 1480 if (rc) 1481 return rc; 1482 } 1483 1484 if (filp->f_mode & FMODE_WRITE) { 1485 rc = mutex_lock_interruptible(&channel->rd_mutex); 1486 if (rc) 1487 goto unlock_wr; 1488 } 1489 1490 if ((filp->f_mode & FMODE_READ) && 1491 (channel->wr_ref_count != 0) && 1492 (channel->wr_exclusive_open)) { 1493 rc = -EBUSY; 1494 goto unlock; 1495 } 1496 1497 if ((filp->f_mode & FMODE_WRITE) && 1498 (channel->rd_ref_count != 0) && 1499 (channel->rd_exclusive_open)) { 1500 rc = -EBUSY; 1501 goto unlock; 1502 } 1503 1504 if (filp->f_mode & FMODE_READ) { 1505 if (channel->wr_ref_count == 0) { /* First open of file */ 1506 /* Move the host to first buffer */ 1507 spin_lock_irqsave(&channel->wr_spinlock, flags); 1508 channel->wr_host_buf_idx = 0; 1509 channel->wr_host_buf_pos = 0; 1510 channel->wr_fpga_buf_idx = -1; 1511 channel->wr_empty = 1; 1512 channel->wr_ready = 0; 1513 channel->wr_sleepy = 1; 1514 channel->wr_eof = -1; 1515 channel->wr_hangup = 0; 1516 1517 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 1518 1519 iowrite32(1 | (channel->chan_num << 1) | 1520 (4 << 24) | /* Opcode 4, open channel */ 1521 ((channel->wr_synchronous & 1) << 23), 1522 channel->endpoint->registers + 1523 fpga_buf_ctrl_reg); 1524 } 1525 1526 channel->wr_ref_count++; 1527 } 1528 1529 if (filp->f_mode & FMODE_WRITE) { 1530 if (channel->rd_ref_count == 0) { /* First open of file */ 1531 /* Move the host to first buffer */ 1532 spin_lock_irqsave(&channel->rd_spinlock, flags); 1533 channel->rd_host_buf_idx = 0; 1534 channel->rd_host_buf_pos = 0; 1535 channel->rd_leftovers[3] = 0; /* No leftovers. */ 1536 channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1; 1537 channel->rd_full = 0; 1538 1539 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1540 1541 iowrite32((channel->chan_num << 1) | 1542 (4 << 24), /* Opcode 4, open channel */ 1543 channel->endpoint->registers + 1544 fpga_buf_ctrl_reg); 1545 } 1546 1547 channel->rd_ref_count++; 1548 } 1549 1550 unlock: 1551 if (filp->f_mode & FMODE_WRITE) 1552 mutex_unlock(&channel->rd_mutex); 1553 unlock_wr: 1554 if (filp->f_mode & FMODE_READ) 1555 mutex_unlock(&channel->wr_mutex); 1556 1557 if (!rc && (!channel->seekable)) 1558 return nonseekable_open(inode, filp); 1559 1560 return rc; 1561 } 1562 1563 static int xillybus_release(struct inode *inode, struct file *filp) 1564 { 1565 unsigned long flags; 1566 struct xilly_channel *channel = filp->private_data; 1567 1568 int buf_idx; 1569 int eof; 1570 1571 if (channel->endpoint->fatal_error) 1572 return -EIO; 1573 1574 if (filp->f_mode & FMODE_WRITE) { 1575 mutex_lock(&channel->rd_mutex); 1576 1577 channel->rd_ref_count--; 1578 1579 if (channel->rd_ref_count == 0) { 1580 /* 1581 * We rely on the kernel calling flush() 1582 * before we get here. 1583 */ 1584 1585 iowrite32((channel->chan_num << 1) | /* Channel ID */ 1586 (5 << 24), /* Opcode 5, close channel */ 1587 channel->endpoint->registers + 1588 fpga_buf_ctrl_reg); 1589 } 1590 mutex_unlock(&channel->rd_mutex); 1591 } 1592 1593 if (filp->f_mode & FMODE_READ) { 1594 mutex_lock(&channel->wr_mutex); 1595 1596 channel->wr_ref_count--; 1597 1598 if (channel->wr_ref_count == 0) { 1599 iowrite32(1 | (channel->chan_num << 1) | 1600 (5 << 24), /* Opcode 5, close channel */ 1601 channel->endpoint->registers + 1602 fpga_buf_ctrl_reg); 1603 1604 /* 1605 * This is crazily cautious: We make sure that not 1606 * only that we got an EOF (be it because we closed 1607 * the channel or because of a user's EOF), but verify 1608 * that it's one beyond the last buffer arrived, so 1609 * we have no leftover buffers pending before wrapping 1610 * up (which can only happen in asynchronous channels, 1611 * BTW) 1612 */ 1613 1614 while (1) { 1615 spin_lock_irqsave(&channel->wr_spinlock, 1616 flags); 1617 buf_idx = channel->wr_fpga_buf_idx; 1618 eof = channel->wr_eof; 1619 channel->wr_sleepy = 1; 1620 spin_unlock_irqrestore(&channel->wr_spinlock, 1621 flags); 1622 1623 /* 1624 * Check if eof points at the buffer after 1625 * the last one the FPGA submitted. Note that 1626 * no EOF is marked by negative eof. 1627 */ 1628 1629 buf_idx++; 1630 if (buf_idx == channel->num_wr_buffers) 1631 buf_idx = 0; 1632 1633 if (buf_idx == eof) 1634 break; 1635 1636 /* 1637 * Steal extra 100 ms if awaken by interrupt. 1638 * This is a simple workaround for an 1639 * interrupt pending when entering, which would 1640 * otherwise result in declaring the hardware 1641 * non-responsive. 1642 */ 1643 1644 if (wait_event_interruptible( 1645 channel->wr_wait, 1646 (!channel->wr_sleepy))) 1647 msleep(100); 1648 1649 if (channel->wr_sleepy) { 1650 mutex_unlock(&channel->wr_mutex); 1651 dev_warn(channel->endpoint->dev, 1652 "Hardware failed to respond to close command, therefore left in messy state.\n"); 1653 return -EINTR; 1654 } 1655 } 1656 } 1657 1658 mutex_unlock(&channel->wr_mutex); 1659 } 1660 1661 return 0; 1662 } 1663 1664 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence) 1665 { 1666 struct xilly_channel *channel = filp->private_data; 1667 loff_t pos = filp->f_pos; 1668 int rc = 0; 1669 1670 /* 1671 * Take both mutexes not allowing interrupts, since it seems like 1672 * common applications don't expect an -EINTR here. Besides, multiple 1673 * access to a single file descriptor on seekable devices is a mess 1674 * anyhow. 1675 */ 1676 1677 if (channel->endpoint->fatal_error) 1678 return -EIO; 1679 1680 mutex_lock(&channel->wr_mutex); 1681 mutex_lock(&channel->rd_mutex); 1682 1683 switch (whence) { 1684 case SEEK_SET: 1685 pos = offset; 1686 break; 1687 case SEEK_CUR: 1688 pos += offset; 1689 break; 1690 case SEEK_END: 1691 pos = offset; /* Going to the end => to the beginning */ 1692 break; 1693 default: 1694 rc = -EINVAL; 1695 goto end; 1696 } 1697 1698 /* In any case, we must finish on an element boundary */ 1699 if (pos & ((1 << channel->log2_element_size) - 1)) { 1700 rc = -EINVAL; 1701 goto end; 1702 } 1703 1704 mutex_lock(&channel->endpoint->register_mutex); 1705 1706 iowrite32(pos >> channel->log2_element_size, 1707 channel->endpoint->registers + fpga_buf_offset_reg); 1708 1709 iowrite32((channel->chan_num << 1) | 1710 (6 << 24), /* Opcode 6, set address */ 1711 channel->endpoint->registers + fpga_buf_ctrl_reg); 1712 1713 mutex_unlock(&channel->endpoint->register_mutex); 1714 1715 end: 1716 mutex_unlock(&channel->rd_mutex); 1717 mutex_unlock(&channel->wr_mutex); 1718 1719 if (rc) /* Return error after releasing mutexes */ 1720 return rc; 1721 1722 filp->f_pos = pos; 1723 1724 /* 1725 * Since seekable devices are allowed only when the channel is 1726 * synchronous, we assume that there is no data pending in either 1727 * direction (which holds true as long as no concurrent access on the 1728 * file descriptor takes place). 1729 * The only thing we may need to throw away is leftovers from partial 1730 * write() flush. 1731 */ 1732 1733 channel->rd_leftovers[3] = 0; 1734 1735 return pos; 1736 } 1737 1738 static __poll_t xillybus_poll(struct file *filp, poll_table *wait) 1739 { 1740 struct xilly_channel *channel = filp->private_data; 1741 __poll_t mask = 0; 1742 unsigned long flags; 1743 1744 poll_wait(filp, &channel->endpoint->ep_wait, wait); 1745 1746 /* 1747 * poll() won't play ball regarding read() channels which 1748 * aren't asynchronous and support the nonempty message. Allowing 1749 * that will create situations where data has been delivered at 1750 * the FPGA, and users expecting select() to wake up, which it may 1751 * not. 1752 */ 1753 1754 if (!channel->wr_synchronous && channel->wr_supports_nonempty) { 1755 poll_wait(filp, &channel->wr_wait, wait); 1756 poll_wait(filp, &channel->wr_ready_wait, wait); 1757 1758 spin_lock_irqsave(&channel->wr_spinlock, flags); 1759 if (!channel->wr_empty || channel->wr_ready) 1760 mask |= EPOLLIN | EPOLLRDNORM; 1761 1762 if (channel->wr_hangup) 1763 /* 1764 * Not EPOLLHUP, because its behavior is in the 1765 * mist, and EPOLLIN does what we want: Wake up 1766 * the read file descriptor so it sees EOF. 1767 */ 1768 mask |= EPOLLIN | EPOLLRDNORM; 1769 spin_unlock_irqrestore(&channel->wr_spinlock, flags); 1770 } 1771 1772 /* 1773 * If partial data write is disallowed on a write() channel, 1774 * it's pointless to ever signal OK to write, because is could 1775 * block despite some space being available. 1776 */ 1777 1778 if (channel->rd_allow_partial) { 1779 poll_wait(filp, &channel->rd_wait, wait); 1780 1781 spin_lock_irqsave(&channel->rd_spinlock, flags); 1782 if (!channel->rd_full) 1783 mask |= EPOLLOUT | EPOLLWRNORM; 1784 spin_unlock_irqrestore(&channel->rd_spinlock, flags); 1785 } 1786 1787 if (channel->endpoint->fatal_error) 1788 mask |= EPOLLERR; 1789 1790 return mask; 1791 } 1792 1793 static const struct file_operations xillybus_fops = { 1794 .owner = THIS_MODULE, 1795 .read = xillybus_read, 1796 .write = xillybus_write, 1797 .open = xillybus_open, 1798 .flush = xillybus_flush, 1799 .release = xillybus_release, 1800 .llseek = xillybus_llseek, 1801 .poll = xillybus_poll, 1802 }; 1803 1804 struct xilly_endpoint *xillybus_init_endpoint(struct device *dev) 1805 { 1806 struct xilly_endpoint *endpoint; 1807 1808 endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL); 1809 if (!endpoint) 1810 return NULL; 1811 1812 endpoint->dev = dev; 1813 endpoint->msg_counter = 0x0b; 1814 endpoint->failed_messages = 0; 1815 endpoint->fatal_error = 0; 1816 1817 init_waitqueue_head(&endpoint->ep_wait); 1818 mutex_init(&endpoint->register_mutex); 1819 1820 return endpoint; 1821 } 1822 EXPORT_SYMBOL(xillybus_init_endpoint); 1823 1824 static int xilly_quiesce(struct xilly_endpoint *endpoint) 1825 { 1826 long t; 1827 1828 endpoint->idtlen = -1; 1829 1830 iowrite32((u32) (endpoint->dma_using_dac & 0x0001), 1831 endpoint->registers + fpga_dma_control_reg); 1832 1833 t = wait_event_interruptible_timeout(endpoint->ep_wait, 1834 (endpoint->idtlen >= 0), 1835 XILLY_TIMEOUT); 1836 if (t <= 0) { 1837 dev_err(endpoint->dev, 1838 "Failed to quiesce the device on exit.\n"); 1839 return -ENODEV; 1840 } 1841 return 0; 1842 } 1843 1844 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint) 1845 { 1846 int rc; 1847 long t; 1848 1849 void *bootstrap_resources; 1850 int idtbuffersize = (1 << PAGE_SHIFT); 1851 struct device *dev = endpoint->dev; 1852 1853 /* 1854 * The bogus IDT is used during bootstrap for allocating the initial 1855 * message buffer, and then the message buffer and space for the IDT 1856 * itself. The initial message buffer is of a single page's size, but 1857 * it's soon replaced with a more modest one (and memory is freed). 1858 */ 1859 1860 unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0, 1861 3, 192, PAGE_SHIFT, 0 }; 1862 struct xilly_idt_handle idt_handle; 1863 1864 /* 1865 * Writing the value 0x00000001 to Endianness register signals which 1866 * endianness this processor is using, so the FPGA can swap words as 1867 * necessary. 1868 */ 1869 1870 iowrite32(1, endpoint->registers + fpga_endian_reg); 1871 1872 /* Bootstrap phase I: Allocate temporary message buffer */ 1873 1874 bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL); 1875 if (!bootstrap_resources) 1876 return -ENOMEM; 1877 1878 endpoint->num_channels = 0; 1879 1880 rc = xilly_setupchannels(endpoint, bogus_idt, 1); 1881 if (rc) 1882 return rc; 1883 1884 /* Clear the message subsystem (and counter in particular) */ 1885 iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg); 1886 1887 endpoint->idtlen = -1; 1888 1889 /* 1890 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT 1891 * buffer size. 1892 */ 1893 iowrite32((u32) (endpoint->dma_using_dac & 0x0001), 1894 endpoint->registers + fpga_dma_control_reg); 1895 1896 t = wait_event_interruptible_timeout(endpoint->ep_wait, 1897 (endpoint->idtlen >= 0), 1898 XILLY_TIMEOUT); 1899 if (t <= 0) { 1900 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n"); 1901 return -ENODEV; 1902 } 1903 1904 /* Enable DMA */ 1905 iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)), 1906 endpoint->registers + fpga_dma_control_reg); 1907 1908 /* Bootstrap phase II: Allocate buffer for IDT and obtain it */ 1909 while (endpoint->idtlen >= idtbuffersize) { 1910 idtbuffersize *= 2; 1911 bogus_idt[6]++; 1912 } 1913 1914 endpoint->num_channels = 1; 1915 1916 rc = xilly_setupchannels(endpoint, bogus_idt, 2); 1917 if (rc) 1918 goto failed_idt; 1919 1920 rc = xilly_obtain_idt(endpoint); 1921 if (rc) 1922 goto failed_idt; 1923 1924 rc = xilly_scan_idt(endpoint, &idt_handle); 1925 if (rc) 1926 goto failed_idt; 1927 1928 devres_close_group(dev, bootstrap_resources); 1929 1930 /* Bootstrap phase III: Allocate buffers according to IDT */ 1931 1932 rc = xilly_setupchannels(endpoint, 1933 idt_handle.chandesc, 1934 idt_handle.entries); 1935 if (rc) 1936 goto failed_idt; 1937 1938 rc = xillybus_init_chrdev(dev, &xillybus_fops, 1939 endpoint->owner, endpoint, 1940 idt_handle.names, 1941 idt_handle.names_len, 1942 endpoint->num_channels, 1943 xillyname, false); 1944 1945 if (rc) 1946 goto failed_idt; 1947 1948 devres_release_group(dev, bootstrap_resources); 1949 1950 return 0; 1951 1952 failed_idt: 1953 xilly_quiesce(endpoint); 1954 flush_workqueue(xillybus_wq); 1955 1956 return rc; 1957 } 1958 EXPORT_SYMBOL(xillybus_endpoint_discovery); 1959 1960 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint) 1961 { 1962 xillybus_cleanup_chrdev(endpoint, endpoint->dev); 1963 1964 xilly_quiesce(endpoint); 1965 1966 /* 1967 * Flushing is done upon endpoint release to prevent access to memory 1968 * just about to be released. This makes the quiesce complete. 1969 */ 1970 flush_workqueue(xillybus_wq); 1971 } 1972 EXPORT_SYMBOL(xillybus_endpoint_remove); 1973 1974 static int __init xillybus_init(void) 1975 { 1976 xillybus_wq = alloc_workqueue(xillyname, 0, 0); 1977 if (!xillybus_wq) 1978 return -ENOMEM; 1979 1980 return 0; 1981 } 1982 1983 static void __exit xillybus_exit(void) 1984 { 1985 /* flush_workqueue() was called for each endpoint released */ 1986 destroy_workqueue(xillybus_wq); 1987 } 1988 1989 module_init(xillybus_init); 1990 module_exit(xillybus_exit); 1991