Lines Matching +full:channel +full:- +full:fifo +full:- +full:len
1 // SPDX-License-Identifier: GPL-2.0-only
8 * a pipe between a hardware FIFO in the programmable logic and a device
76 int fill; /* Number of bytes in the FIFO */
107 struct xillyfifo fifo; member
161 unsigned int len; member
219 * calls to each on the same FIFO is not allowed) however it's OK to have
220 * threads calling each of the two functions once on the same FIFO, and
224 static int fifo_write(struct xillyfifo *fifo, in fifo_write() argument
225 const void *data, unsigned int len, in fifo_write() argument
229 unsigned int todo = len; in fifo_write()
231 unsigned int writepos = fifo->writepos; in fifo_write()
232 unsigned int writebuf = fifo->writebuf; in fifo_write()
236 nmax = fifo->size - READ_ONCE(fifo->fill); in fifo_write()
239 unsigned int nrail = fifo->bufsize - writepos; in fifo_write()
243 spin_lock_irqsave(&fifo->lock, flags); in fifo_write()
244 fifo->fill += done; in fifo_write()
245 spin_unlock_irqrestore(&fifo->lock, flags); in fifo_write()
247 fifo->writepos = writepos; in fifo_write()
248 fifo->writebuf = writebuf; in fifo_write()
256 rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n); in fifo_write()
262 todo -= n; in fifo_write()
265 nmax -= n; in fifo_write()
267 if (writepos == fifo->bufsize) { in fifo_write()
271 if (writebuf == fifo->bufnum) in fifo_write()
277 static int fifo_read(struct xillyfifo *fifo, in fifo_read() argument
278 void *data, unsigned int len, in fifo_read() argument
282 unsigned int todo = len; in fifo_read()
284 unsigned int readpos = fifo->readpos; in fifo_read()
285 unsigned int readbuf = fifo->readbuf; in fifo_read()
290 * The spinlock here is necessary, because otherwise fifo->fill in fifo_read()
293 * visible on this thread at the time the updated fifo->fill was. in fifo_read()
297 spin_lock_irqsave(&fifo->lock, flags); in fifo_read()
298 fill = fifo->fill; in fifo_read()
299 spin_unlock_irqrestore(&fifo->lock, flags); in fifo_read()
302 unsigned int nrail = fifo->bufsize - readpos; in fifo_read()
306 spin_lock_irqsave(&fifo->lock, flags); in fifo_read()
307 fifo->fill -= done; in fifo_read()
308 spin_unlock_irqrestore(&fifo->lock, flags); in fifo_read()
310 fifo->readpos = readpos; in fifo_read()
311 fifo->readbuf = readbuf; in fifo_read()
319 rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n); in fifo_read()
325 todo -= n; in fifo_read()
328 fill -= n; in fifo_read()
330 if (readpos == fifo->bufsize) { in fifo_read()
334 if (readbuf == fifo->bufnum) in fifo_read()
349 return -EFAULT; in xilly_copy_from_user()
357 return -EFAULT; in xilly_copy_to_user()
369 static int fifo_init(struct xillyfifo *fifo, in fifo_init() argument
382 log2_bufnum = log2_size - log2_fifo_buf_size; in fifo_init()
384 fifo->bufsize = 1 << log2_fifo_buf_size; in fifo_init()
388 log2_size - PAGE_SHIFT : 0; in fifo_init()
389 fifo->bufsize = 1 << log2_size; in fifo_init()
392 fifo->bufnum = 1 << log2_bufnum; in fifo_init()
393 fifo->size = fifo->bufnum * fifo->bufsize; in fifo_init()
394 fifo->buf_order = buf_order; in fifo_init()
396 fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL); in fifo_init()
398 if (!fifo->mem) in fifo_init()
399 return -ENOMEM; in fifo_init()
401 for (i = 0; i < fifo->bufnum; i++) { in fifo_init()
402 fifo->mem[i] = (void *) in fifo_init()
405 if (!fifo->mem[i]) in fifo_init()
409 fifo->fill = 0; in fifo_init()
410 fifo->readpos = 0; in fifo_init()
411 fifo->readbuf = 0; in fifo_init()
412 fifo->writepos = 0; in fifo_init()
413 fifo->writebuf = 0; in fifo_init()
414 spin_lock_init(&fifo->lock); in fifo_init()
415 init_waitqueue_head(&fifo->waitq); in fifo_init()
419 for (i--; i >= 0; i--) in fifo_init()
420 free_pages((unsigned long)fifo->mem[i], buf_order); in fifo_init()
422 kfree(fifo->mem); in fifo_init()
423 fifo->mem = NULL; in fifo_init()
426 fifo_buf_order--; in fifo_init()
429 return -ENOMEM; in fifo_init()
433 static void fifo_mem_release(struct xillyfifo *fifo) in fifo_mem_release() argument
437 if (!fifo->mem) in fifo_mem_release()
440 for (i = 0; i < fifo->bufnum; i++) in fifo_mem_release()
441 free_pages((unsigned long)fifo->mem[i], fifo->buf_order); in fifo_mem_release()
443 kfree(fifo->mem); in fifo_mem_release()
454 mutex_lock(&ep->ep_mutex); in endpoint_quiesce()
455 ep->shutting_down = true; in endpoint_quiesce()
456 mutex_unlock(&ep->ep_mutex); in endpoint_quiesce()
458 usb_kill_anchored_urbs(&ep->anchor); in endpoint_quiesce()
459 cancel_work_sync(&ep->workitem); in endpoint_quiesce()
463 * Note that endpoint_dealloc() also frees fifo memory (if allocated), even
471 fifo_mem_release(&ep->fifo); in endpoint_dealloc()
474 list_splice(&ep->filled_buffers, &ep->buffers); in endpoint_dealloc()
476 list_for_each_safe(this, next, &ep->buffers) { in endpoint_dealloc()
480 free_pages((unsigned long)xb->buf, ep->order); in endpoint_dealloc()
503 INIT_LIST_HEAD(&ep->buffers); in endpoint_alloc()
504 INIT_LIST_HEAD(&ep->filled_buffers); in endpoint_alloc()
506 spin_lock_init(&ep->buffers_lock); in endpoint_alloc()
507 mutex_init(&ep->ep_mutex); in endpoint_alloc()
509 init_usb_anchor(&ep->anchor); in endpoint_alloc()
510 INIT_WORK(&ep->workitem, work); in endpoint_alloc()
512 ep->order = order; in endpoint_alloc()
513 ep->buffer_size = 1 << (PAGE_SHIFT + order); in endpoint_alloc()
514 ep->outstanding_urbs = 0; in endpoint_alloc()
515 ep->drained = true; in endpoint_alloc()
516 ep->wake_on_drain = false; in endpoint_alloc()
517 ep->xdev = xdev; in endpoint_alloc()
518 ep->ep_num = ep_num; in endpoint_alloc()
519 ep->shutting_down = false; in endpoint_alloc()
540 xb->buf = (void *)addr; in endpoint_alloc()
541 xb->ep = ep; in endpoint_alloc()
542 list_add_tail(&xb->entry, &ep->buffers); in endpoint_alloc()
552 if (xdev->in_ep) in cleanup_dev()
553 endpoint_dealloc(xdev->in_ep); in cleanup_dev()
555 if (xdev->msg_ep) in cleanup_dev()
556 endpoint_dealloc(xdev->msg_ep); in cleanup_dev()
558 if (xdev->workq) in cleanup_dev()
559 destroy_workqueue(xdev->workq); in cleanup_dev()
561 usb_put_dev(xdev->udev); in cleanup_dev()
562 kfree(xdev->channels); /* Argument may be NULL, and that's fine */ in cleanup_dev()
570 * errors if executed. The mechanism relies on that xdev->error is assigned
571 * a non-zero value by report_io_error() prior to queueing wakeup_all(),
581 mutex_lock(&xdev->process_in_mutex); in wakeup_all()
583 for (i = 0; i < xdev->num_channels; i++) { in wakeup_all()
584 struct xillyusb_channel *chan = &xdev->channels[i]; in wakeup_all()
586 mutex_lock(&chan->lock); in wakeup_all()
588 if (chan->in_fifo) { in wakeup_all()
593 chan->read_data_ok = 0; in wakeup_all()
594 wake_up_interruptible(&chan->in_fifo->waitq); in wakeup_all()
597 if (chan->out_ep) in wakeup_all()
598 wake_up_interruptible(&chan->out_ep->fifo.waitq); in wakeup_all()
600 mutex_unlock(&chan->lock); in wakeup_all()
602 wake_up_interruptible(&chan->flushq); in wakeup_all()
605 mutex_unlock(&xdev->process_in_mutex); in wakeup_all()
607 wake_up_interruptible(&xdev->msg_ep->fifo.waitq); in wakeup_all()
609 kref_put(&xdev->kref, cleanup_dev); in wakeup_all()
618 spin_lock_irqsave(&xdev->error_lock, flags); in report_io_error()
619 if (!xdev->error) { in report_io_error()
620 xdev->error = errcode; in report_io_error()
623 spin_unlock_irqrestore(&xdev->error_lock, flags); in report_io_error()
626 kref_get(&xdev->kref); /* xdev is used by work item */ in report_io_error()
627 queue_work(wakeup_wq, &xdev->wakeup_workitem); in report_io_error()
632 * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures
637 struct xillyfifo *fifo) in safely_assign_in_fifo() argument
639 mutex_lock(&chan->lock); in safely_assign_in_fifo()
640 chan->in_fifo = fifo; in safely_assign_in_fifo()
641 mutex_unlock(&chan->lock); in safely_assign_in_fifo()
643 flush_work(&chan->xdev->in_ep->workitem); in safely_assign_in_fifo()
648 struct xillybuffer *xb = urb->context; in bulk_in_completer()
649 struct xillyusb_endpoint *ep = xb->ep; in bulk_in_completer()
652 if (urb->status) { in bulk_in_completer()
653 if (!(urb->status == -ENOENT || in bulk_in_completer()
654 urb->status == -ECONNRESET || in bulk_in_completer()
655 urb->status == -ESHUTDOWN)) in bulk_in_completer()
656 report_io_error(ep->xdev, -EIO); in bulk_in_completer()
658 spin_lock_irqsave(&ep->buffers_lock, flags); in bulk_in_completer()
659 list_add_tail(&xb->entry, &ep->buffers); in bulk_in_completer()
660 ep->outstanding_urbs--; in bulk_in_completer()
661 spin_unlock_irqrestore(&ep->buffers_lock, flags); in bulk_in_completer()
666 xb->len = urb->actual_length; in bulk_in_completer()
668 spin_lock_irqsave(&ep->buffers_lock, flags); in bulk_in_completer()
669 list_add_tail(&xb->entry, &ep->filled_buffers); in bulk_in_completer()
670 spin_unlock_irqrestore(&ep->buffers_lock, flags); in bulk_in_completer()
672 if (!ep->shutting_down) in bulk_in_completer()
673 queue_work(ep->xdev->workq, &ep->workitem); in bulk_in_completer()
678 struct xillybuffer *xb = urb->context; in bulk_out_completer()
679 struct xillyusb_endpoint *ep = xb->ep; in bulk_out_completer()
682 if (urb->status && in bulk_out_completer()
683 (!(urb->status == -ENOENT || in bulk_out_completer()
684 urb->status == -ECONNRESET || in bulk_out_completer()
685 urb->status == -ESHUTDOWN))) in bulk_out_completer()
686 report_io_error(ep->xdev, -EIO); in bulk_out_completer()
688 spin_lock_irqsave(&ep->buffers_lock, flags); in bulk_out_completer()
689 list_add_tail(&xb->entry, &ep->buffers); in bulk_out_completer()
690 ep->outstanding_urbs--; in bulk_out_completer()
691 spin_unlock_irqrestore(&ep->buffers_lock, flags); in bulk_out_completer()
693 if (!ep->shutting_down) in bulk_out_completer()
694 queue_work(ep->xdev->workq, &ep->workitem); in bulk_out_completer()
699 struct xillyusb_dev *xdev = ep->xdev; in try_queue_bulk_in()
705 unsigned int bufsize = ep->buffer_size; in try_queue_bulk_in()
707 mutex_lock(&ep->ep_mutex); in try_queue_bulk_in()
709 if (ep->shutting_down || xdev->error) in try_queue_bulk_in()
713 spin_lock_irqsave(&ep->buffers_lock, flags); in try_queue_bulk_in()
715 if (list_empty(&ep->buffers)) { in try_queue_bulk_in()
716 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_in()
720 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry); in try_queue_bulk_in()
721 list_del(&xb->entry); in try_queue_bulk_in()
722 ep->outstanding_urbs++; in try_queue_bulk_in()
724 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_in()
728 report_io_error(xdev, -ENOMEM); in try_queue_bulk_in()
732 usb_fill_bulk_urb(urb, xdev->udev, in try_queue_bulk_in()
733 usb_rcvbulkpipe(xdev->udev, ep->ep_num), in try_queue_bulk_in()
734 xb->buf, bufsize, bulk_in_completer, xb); in try_queue_bulk_in()
736 usb_anchor_urb(urb, &ep->anchor); in try_queue_bulk_in()
741 report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM : in try_queue_bulk_in()
742 -EIO); in try_queue_bulk_in()
754 spin_lock_irqsave(&ep->buffers_lock, flags); in try_queue_bulk_in()
755 list_add_tail(&xb->entry, &ep->buffers); in try_queue_bulk_in()
756 ep->outstanding_urbs--; in try_queue_bulk_in()
757 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_in()
760 mutex_unlock(&ep->ep_mutex); in try_queue_bulk_in()
765 struct xillyfifo *fifo = &ep->fifo; in try_queue_bulk_out() local
766 struct xillyusb_dev *xdev = ep->xdev; in try_queue_bulk_out()
775 mutex_lock(&ep->ep_mutex); in try_queue_bulk_out()
777 if (ep->shutting_down || xdev->error) in try_queue_bulk_out()
780 fill = READ_ONCE(fifo->fill) & ep->fill_mask; in try_queue_bulk_out()
786 spin_lock_irqsave(&ep->buffers_lock, flags); in try_queue_bulk_out()
789 * Race conditions might have the FIFO filled while the in try_queue_bulk_out()
792 * certain data has been sent on the USB channel before in try_queue_bulk_out()
793 * shutting it down. Hence knowing that the FIFO appears in try_queue_bulk_out()
799 ep->drained = !ep->outstanding_urbs; in try_queue_bulk_out()
800 if (ep->drained && ep->wake_on_drain) in try_queue_bulk_out()
803 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_out()
807 ep->drained = false; in try_queue_bulk_out()
809 if ((fill < ep->buffer_size && ep->outstanding_urbs) || in try_queue_bulk_out()
810 list_empty(&ep->buffers)) { in try_queue_bulk_out()
811 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_out()
815 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry); in try_queue_bulk_out()
816 list_del(&xb->entry); in try_queue_bulk_out()
817 ep->outstanding_urbs++; in try_queue_bulk_out()
819 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_out()
821 max_read = min(fill, ep->buffer_size); in try_queue_bulk_out()
823 count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy); in try_queue_bulk_out()
832 report_io_error(xdev, -ENOMEM); in try_queue_bulk_out()
836 usb_fill_bulk_urb(urb, xdev->udev, in try_queue_bulk_out()
837 usb_sndbulkpipe(xdev->udev, ep->ep_num), in try_queue_bulk_out()
838 xb->buf, count, bulk_out_completer, xb); in try_queue_bulk_out()
840 usb_anchor_urb(urb, &ep->anchor); in try_queue_bulk_out()
845 report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM : in try_queue_bulk_out()
846 -EIO); in try_queue_bulk_out()
852 fill -= count; in try_queue_bulk_out()
861 spin_lock_irqsave(&ep->buffers_lock, flags); in try_queue_bulk_out()
862 list_add_tail(&xb->entry, &ep->buffers); in try_queue_bulk_out()
863 ep->outstanding_urbs--; in try_queue_bulk_out()
864 spin_unlock_irqrestore(&ep->buffers_lock, flags); in try_queue_bulk_out()
867 mutex_unlock(&ep->ep_mutex); in try_queue_bulk_out()
870 wake_up_interruptible(&fifo->waitq); in try_queue_bulk_out()
886 struct device *dev = xdev->dev; in process_in_opcode()
889 if (chan_idx >= xdev->num_channels) { in process_in_opcode()
890 dev_err(dev, "Received illegal channel ID %d from FPGA\n", in process_in_opcode()
892 return -EIO; in process_in_opcode()
895 chan = &xdev->channels[chan_idx]; in process_in_opcode()
899 if (!chan->read_data_ok) { in process_in_opcode()
900 dev_err(dev, "Received unexpected EOF for channel %d\n", in process_in_opcode()
902 return -EIO; in process_in_opcode()
906 * A write memory barrier ensures that the FIFO's fill level in process_in_opcode()
908 * the FIFO isn't missed by the consumer. in process_in_opcode()
911 WRITE_ONCE(chan->read_data_ok, 0); in process_in_opcode()
912 wake_up_interruptible(&chan->in_fifo->waitq); in process_in_opcode()
916 chan->flushing = 0; in process_in_opcode()
917 wake_up_interruptible(&chan->flushq); in process_in_opcode()
921 chan->canceled = 1; in process_in_opcode()
922 wake_up_interruptible(&chan->flushq); in process_in_opcode()
928 return -EIO; in process_in_opcode()
936 struct xillyusb_endpoint *ep = xb->ep; in process_bulk_in()
937 struct xillyusb_dev *xdev = ep->xdev; in process_bulk_in()
938 struct device *dev = xdev->dev; in process_bulk_in()
939 int dws = xb->len >> 2; in process_bulk_in()
940 __le32 *p = xb->buf; in process_bulk_in()
943 struct xillyfifo *fifo; in process_bulk_in() local
950 if ((dws << 2) != xb->len) { in process_bulk_in()
952 xb->len); in process_bulk_in()
953 return -EIO; in process_bulk_in()
956 if (xdev->in_bytes_left) { in process_bulk_in()
957 bytes = min(xdev->in_bytes_left, dws << 2); in process_bulk_in()
958 in_bytes_left = xdev->in_bytes_left - bytes; in process_bulk_in()
959 chan_num = xdev->leftover_chan_num; in process_bulk_in()
965 dws--; in process_bulk_in()
972 unsigned int in_counter = xdev->in_counter++ & 0x3ff; in process_bulk_in()
977 return -EIO; in process_bulk_in()
989 in_bytes_left = count + 1 - bytes; in process_bulk_in()
994 if (!(chan_num & 1) || chan_idx >= xdev->num_channels || in process_bulk_in()
995 !xdev->channels[chan_idx].read_data_ok) { in process_bulk_in()
996 dev_err(dev, "Received illegal channel ID %d from FPGA\n", in process_bulk_in()
998 return -EIO; in process_bulk_in()
1000 chan = &xdev->channels[chan_idx]; in process_bulk_in()
1002 fifo = chan->in_fifo; in process_bulk_in()
1004 if (unlikely(!fifo)) in process_bulk_in()
1005 return -EIO; /* We got really unexpected data */ in process_bulk_in()
1007 if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) { in process_bulk_in()
1008 dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n"); in process_bulk_in()
1009 return -EIO; in process_bulk_in()
1012 wake_up_interruptible(&fifo->waitq); in process_bulk_in()
1015 dws -= dwconsume; in process_bulk_in()
1019 xdev->in_bytes_left = in_bytes_left; in process_bulk_in()
1020 xdev->leftover_chan_num = chan_num; in process_bulk_in()
1028 struct xillyusb_dev *xdev = ep->xdev; in bulk_in_work()
1034 mutex_lock(&xdev->process_in_mutex); in bulk_in_work()
1036 spin_lock_irqsave(&ep->buffers_lock, flags); in bulk_in_work()
1039 if (rc || list_empty(&ep->filled_buffers)) { in bulk_in_work()
1040 spin_unlock_irqrestore(&ep->buffers_lock, flags); in bulk_in_work()
1041 mutex_unlock(&xdev->process_in_mutex); in bulk_in_work()
1051 xb = list_first_entry(&ep->filled_buffers, struct xillybuffer, in bulk_in_work()
1053 list_del(&xb->entry); in bulk_in_work()
1055 spin_unlock_irqrestore(&ep->buffers_lock, flags); in bulk_in_work()
1059 if (!xdev->error) in bulk_in_work()
1062 spin_lock_irqsave(&ep->buffers_lock, flags); in bulk_in_work()
1063 list_add_tail(&xb->entry, &ep->buffers); in bulk_in_work()
1064 ep->outstanding_urbs--; in bulk_in_work()
1071 struct xillyusb_endpoint *ep = xdev->msg_ep; in xillyusb_send_opcode()
1072 struct xillyfifo *fifo = &ep->fifo; in xillyusb_send_opcode() local
1081 mutex_lock(&xdev->msg_mutex); in xillyusb_send_opcode()
1087 * the release method. And the xdev->error part prevents being stuck in xillyusb_send_opcode()
1091 while (wait_event_interruptible(fifo->waitq, in xillyusb_send_opcode()
1092 fifo->fill <= (fifo->size - 8) || in xillyusb_send_opcode()
1093 xdev->error)) in xillyusb_send_opcode()
1096 if (xdev->error) { in xillyusb_send_opcode()
1097 rc = xdev->error; in xillyusb_send_opcode()
1101 fifo_write(fifo, (void *)msg, 8, xilly_memcpy); in xillyusb_send_opcode()
1106 mutex_unlock(&xdev->msg_mutex); in xillyusb_send_opcode()
1113 * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart,
1121 * chan->flushed is there to avoid multiple flushes at the same position,
1130 struct xillyusb_dev *xdev = chan->xdev; in flush_downstream()
1131 int chan_num = chan->chan_idx << 1; in flush_downstream()
1135 if (chan->flushed) in flush_downstream()
1140 if (chan->flushing) { in flush_downstream()
1143 chan->canceled = 0; in flush_downstream()
1148 return rc; /* Only real error, never -EINTR */ in flush_downstream()
1151 while (!chan->canceled) { in flush_downstream()
1152 left_to_sleep = cancel_deadline - ((long)jiffies); in flush_downstream()
1155 report_io_error(xdev, -EIO); in flush_downstream()
1156 return -EIO; in flush_downstream()
1159 rc = wait_event_interruptible_timeout(chan->flushq, in flush_downstream()
1160 chan->canceled || in flush_downstream()
1161 xdev->error, in flush_downstream()
1164 if (xdev->error) in flush_downstream()
1165 return xdev->error; in flush_downstream()
1169 chan->flushing = 1; in flush_downstream()
1174 * FIFO, it's not flushed, including the flush before closing, which in flush_downstream()
1180 chan->out_bytes >> in flush_downstream()
1181 chan->out_log2_element_size); in flush_downstream()
1184 return rc; /* Only real error, never -EINTR */ in flush_downstream()
1187 while (chan->flushing) { in flush_downstream()
1188 rc = wait_event_interruptible(chan->flushq, in flush_downstream()
1189 !chan->flushing || in flush_downstream()
1190 xdev->error); in flush_downstream()
1191 if (xdev->error) in flush_downstream()
1192 return xdev->error; in flush_downstream()
1195 return -EINTR; in flush_downstream()
1201 while (chan->flushing) { in flush_downstream()
1202 left_to_sleep = deadline - ((long)jiffies); in flush_downstream()
1205 return -ETIMEDOUT; in flush_downstream()
1207 rc = wait_event_interruptible_timeout(chan->flushq, in flush_downstream()
1208 !chan->flushing || in flush_downstream()
1209 xdev->error, in flush_downstream()
1212 if (xdev->error) in flush_downstream()
1213 return xdev->error; in flush_downstream()
1216 return -EINTR; in flush_downstream()
1220 chan->flushed = 1; in flush_downstream()
1228 struct xillyusb_dev *xdev = chan->xdev; in request_read_anything()
1229 unsigned int sh = chan->in_log2_element_size; in request_read_anything()
1230 int chan_num = (chan->chan_idx << 1) | 1; in request_read_anything()
1231 u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1; in request_read_anything()
1253 kref_get(&xdev->kref); in xillyusb_open()
1256 chan = &xdev->channels[index]; in xillyusb_open()
1257 filp->private_data = chan; in xillyusb_open()
1259 mutex_lock(&chan->lock); in xillyusb_open()
1261 rc = -ENODEV; in xillyusb_open()
1263 if (xdev->error) in xillyusb_open()
1266 if (((filp->f_mode & FMODE_READ) && !chan->readable) || in xillyusb_open()
1267 ((filp->f_mode & FMODE_WRITE) && !chan->writable)) in xillyusb_open()
1270 if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) && in xillyusb_open()
1271 chan->in_synchronous) { in xillyusb_open()
1272 dev_err(xdev->dev, in xillyusb_open()
1277 if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) && in xillyusb_open()
1278 chan->out_synchronous) { in xillyusb_open()
1279 dev_err(xdev->dev, in xillyusb_open()
1284 rc = -EBUSY; in xillyusb_open()
1286 if (((filp->f_mode & FMODE_READ) && chan->open_for_read) || in xillyusb_open()
1287 ((filp->f_mode & FMODE_WRITE) && chan->open_for_write)) in xillyusb_open()
1290 if (filp->f_mode & FMODE_READ) in xillyusb_open()
1291 chan->open_for_read = 1; in xillyusb_open()
1293 if (filp->f_mode & FMODE_WRITE) in xillyusb_open()
1294 chan->open_for_write = 1; in xillyusb_open()
1296 mutex_unlock(&chan->lock); in xillyusb_open()
1298 if (filp->f_mode & FMODE_WRITE) { in xillyusb_open()
1300 (chan->chan_idx + 2) | USB_DIR_OUT, in xillyusb_open()
1304 rc = -ENOMEM; in xillyusb_open()
1308 rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size); in xillyusb_open()
1313 out_ep->fill_mask = -(1 << chan->out_log2_element_size); in xillyusb_open()
1314 chan->out_bytes = 0; in xillyusb_open()
1315 chan->flushed = 0; in xillyusb_open()
1329 if (rc == -ETIMEDOUT) { in xillyusb_open()
1330 rc = -EIO; in xillyusb_open()
1338 if (filp->f_mode & FMODE_READ) { in xillyusb_open()
1342 rc = -ENOMEM; in xillyusb_open()
1346 rc = fifo_init(in_fifo, chan->in_log2_fifo_size); in xillyusb_open()
1354 mutex_lock(&chan->lock); in xillyusb_open()
1356 chan->in_fifo = in_fifo; in xillyusb_open()
1357 chan->read_data_ok = 1; in xillyusb_open()
1360 chan->out_ep = out_ep; in xillyusb_open()
1361 mutex_unlock(&chan->lock); in xillyusb_open()
1366 if (!chan->in_synchronous) in xillyusb_open()
1367 in_checkpoint = in_fifo->size >> in xillyusb_open()
1368 chan->in_log2_element_size; in xillyusb_open()
1370 chan->in_consumed_bytes = 0; in xillyusb_open()
1371 chan->poll_used = 0; in xillyusb_open()
1372 chan->in_current_checkpoint = in_checkpoint; in xillyusb_open()
1373 rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1, in xillyusb_open()
1381 * In non-blocking mode, request the FPGA to send any data it in xillyusb_open()
1383 * return -EAGAIN, which is OK strictly speaking, but ugly. in xillyusb_open()
1385 * effort -- the error is propagated to the first read() in xillyusb_open()
1388 if (filp->f_flags & O_NONBLOCK) in xillyusb_open()
1395 chan->read_data_ok = 0; in xillyusb_open()
1401 mutex_lock(&chan->lock); in xillyusb_open()
1402 chan->out_ep = NULL; in xillyusb_open()
1403 mutex_unlock(&chan->lock); in xillyusb_open()
1411 mutex_lock(&chan->lock); in xillyusb_open()
1413 if (filp->f_mode & FMODE_READ) in xillyusb_open()
1414 chan->open_for_read = 0; in xillyusb_open()
1416 if (filp->f_mode & FMODE_WRITE) in xillyusb_open()
1417 chan->open_for_write = 0; in xillyusb_open()
1419 mutex_unlock(&chan->lock); in xillyusb_open()
1421 kref_put(&xdev->kref, cleanup_dev); in xillyusb_open()
1426 kref_put(&xdev->kref, cleanup_dev); in xillyusb_open()
1427 mutex_unlock(&chan->lock); in xillyusb_open()
1434 struct xillyusb_channel *chan = filp->private_data; in xillyusb_read()
1435 struct xillyusb_dev *xdev = chan->xdev; in xillyusb_read()
1436 struct xillyfifo *fifo = chan->in_fifo; in xillyusb_read() local
1437 int chan_num = (chan->chan_idx << 1) | 1; in xillyusb_read()
1446 rc = mutex_lock_interruptible(&chan->in_mutex); in xillyusb_read()
1456 unsigned int sh = chan->in_log2_element_size; in xillyusb_read()
1459 rc = fifo_read(fifo, (__force void *)userbuf + bytes_done, in xillyusb_read()
1460 count - bytes_done, xilly_copy_to_user); in xillyusb_read()
1466 chan->in_consumed_bytes += rc; in xillyusb_read()
1468 left_to_sleep = deadline - ((long)jiffies); in xillyusb_read()
1471 * Some 32-bit arithmetic that may wrap. Note that in xillyusb_read()
1478 fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size; in xillyusb_read()
1480 chan->in_consumed_bytes + count - bytes_done; in xillyusb_read()
1484 (complete_checkpoint_bytes + (1 << sh) - 1) >> sh; in xillyusb_read()
1486 diff = (fifo_checkpoint - complete_checkpoint) << sh; in xillyusb_read()
1488 if (chan->in_synchronous && diff >= 0) { in xillyusb_read()
1496 leap = (checkpoint - chan->in_current_checkpoint) << sh; in xillyusb_read()
1501 * checkpoint by at least an 8th of the FIFO's size, or if in xillyusb_read()
1505 * chan->read_data_ok is checked to spare an unnecessary in xillyusb_read()
1510 if (chan->read_data_ok && in xillyusb_read()
1511 (leap > (fifo->size >> 3) || in xillyusb_read()
1513 chan->in_current_checkpoint = checkpoint; in xillyusb_read()
1527 * Reaching here means that the FIFO was empty when in xillyusb_read()
1530 * that managed its way to the FIFO is lost. in xillyusb_read()
1533 if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */ in xillyusb_read()
1534 /* Has data slipped into the FIFO since fifo_read()? */ in xillyusb_read()
1536 if (READ_ONCE(fifo->fill)) in xillyusb_read()
1543 if (xdev->error) { in xillyusb_read()
1544 rc = xdev->error; in xillyusb_read()
1548 if (filp->f_flags & O_NONBLOCK) { in xillyusb_read()
1549 rc = -EAGAIN; in xillyusb_read()
1566 * Note that when xdev->error is set (e.g. when the in xillyusb_read()
1568 * fifo->waitq is awaken. in xillyusb_read()
1569 * Therefore no special attention to xdev->error. in xillyusb_read()
1573 (fifo->waitq, in xillyusb_read()
1574 fifo->fill || !chan->read_data_ok, in xillyusb_read()
1584 (fifo->waitq, in xillyusb_read()
1585 fifo->fill || !chan->read_data_ok); in xillyusb_read()
1589 rc = -EINTR; in xillyusb_read()
1594 if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) && in xillyusb_read()
1595 !READ_ONCE(fifo->fill)) in xillyusb_read()
1598 mutex_unlock(&chan->in_mutex); in xillyusb_read()
1608 struct xillyusb_channel *chan = filp->private_data; in xillyusb_flush()
1611 if (!(filp->f_mode & FMODE_WRITE)) in xillyusb_flush()
1614 rc = mutex_lock_interruptible(&chan->out_mutex); in xillyusb_flush()
1621 * the user pressed CTRL-C, that interrupt will still be in flight by in xillyusb_flush()
1626 mutex_unlock(&chan->out_mutex); in xillyusb_flush()
1628 if (rc == -ETIMEDOUT) { in xillyusb_flush()
1630 struct xillyusb_dev *xdev = chan->xdev; in xillyusb_flush()
1632 mutex_lock(&chan->lock); in xillyusb_flush()
1633 if (!xdev->error) in xillyusb_flush()
1634 dev_warn(xdev->dev, in xillyusb_flush()
1636 mutex_unlock(&chan->lock); in xillyusb_flush()
1645 struct xillyusb_channel *chan = filp->private_data; in xillyusb_write()
1646 struct xillyusb_dev *xdev = chan->xdev; in xillyusb_write()
1647 struct xillyfifo *fifo = &chan->out_ep->fifo; in xillyusb_write() local
1650 rc = mutex_lock_interruptible(&chan->out_mutex); in xillyusb_write()
1656 if (xdev->error) { in xillyusb_write()
1657 rc = xdev->error; in xillyusb_write()
1664 rc = fifo_write(fifo, (__force void *)userbuf, count, in xillyusb_write()
1670 if (filp->f_flags & O_NONBLOCK) { in xillyusb_write()
1671 rc = -EAGAIN; in xillyusb_write()
1676 (fifo->waitq, in xillyusb_write()
1677 fifo->fill != fifo->size || xdev->error)) { in xillyusb_write()
1678 rc = -EINTR; in xillyusb_write()
1686 chan->out_bytes += rc; in xillyusb_write()
1689 try_queue_bulk_out(chan->out_ep); in xillyusb_write()
1690 chan->flushed = 0; in xillyusb_write()
1693 if (chan->out_synchronous) { in xillyusb_write()
1701 mutex_unlock(&chan->out_mutex); in xillyusb_write()
1708 struct xillyusb_channel *chan = filp->private_data; in xillyusb_release()
1709 struct xillyusb_dev *xdev = chan->xdev; in xillyusb_release()
1712 if (filp->f_mode & FMODE_READ) { in xillyusb_release()
1713 struct xillyfifo *in_fifo = chan->in_fifo; in xillyusb_release()
1715 rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1, in xillyusb_release()
1718 * If rc_read is nonzero, xdev->error indicates a global in xillyusb_release()
1726 * there's a global device error, chan->read_data_ok is in xillyusb_release()
1730 while (wait_event_interruptible(in_fifo->waitq, in xillyusb_release()
1731 !chan->read_data_ok)) in xillyusb_release()
1738 mutex_lock(&chan->lock); in xillyusb_release()
1739 chan->open_for_read = 0; in xillyusb_release()
1740 mutex_unlock(&chan->lock); in xillyusb_release()
1743 if (filp->f_mode & FMODE_WRITE) { in xillyusb_release()
1744 struct xillyusb_endpoint *ep = chan->out_ep; in xillyusb_release()
1746 * chan->flushing isn't zeroed. If the pre-release flush timed in xillyusb_release()
1753 * file has re-opened. in xillyusb_release()
1756 mutex_lock(&chan->lock); in xillyusb_release()
1757 chan->out_ep = NULL; in xillyusb_release()
1758 mutex_unlock(&chan->lock); in xillyusb_release()
1764 rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1, in xillyusb_release()
1767 mutex_lock(&chan->lock); in xillyusb_release()
1768 chan->open_for_write = 0; in xillyusb_release()
1769 mutex_unlock(&chan->lock); in xillyusb_release()
1772 kref_put(&xdev->kref, cleanup_dev); in xillyusb_release()
1784 struct xillyusb_channel *chan = filp->private_data; in xillyusb_llseek()
1785 struct xillyusb_dev *xdev = chan->xdev; in xillyusb_llseek()
1786 loff_t pos = filp->f_pos; in xillyusb_llseek()
1788 unsigned int log2_element_size = chan->readable ? in xillyusb_llseek()
1789 chan->in_log2_element_size : chan->out_log2_element_size; in xillyusb_llseek()
1793 * common applications don't expect an -EINTR here. Besides, multiple in xillyusb_llseek()
1798 mutex_lock(&chan->out_mutex); in xillyusb_llseek()
1799 mutex_lock(&chan->in_mutex); in xillyusb_llseek()
1812 rc = -EINVAL; in xillyusb_llseek()
1817 if (pos & ((1 << log2_element_size) - 1)) { in xillyusb_llseek()
1818 rc = -EINVAL; in xillyusb_llseek()
1822 rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1, in xillyusb_llseek()
1829 if (chan->writable) { in xillyusb_llseek()
1830 chan->flushed = 0; in xillyusb_llseek()
1835 mutex_unlock(&chan->out_mutex); in xillyusb_llseek()
1836 mutex_unlock(&chan->in_mutex); in xillyusb_llseek()
1841 filp->f_pos = pos; in xillyusb_llseek()
1848 struct xillyusb_channel *chan = filp->private_data; in xillyusb_poll()
1851 if (chan->in_fifo) in xillyusb_poll()
1852 poll_wait(filp, &chan->in_fifo->waitq, wait); in xillyusb_poll()
1854 if (chan->out_ep) in xillyusb_poll()
1855 poll_wait(filp, &chan->out_ep->fifo.waitq, wait); in xillyusb_poll()
1861 * data in in_fifo unless the stream is dry end-to-end. Note that the in xillyusb_poll()
1867 if (!chan->poll_used && chan->in_fifo) { in xillyusb_poll()
1868 chan->poll_used = 1; in xillyusb_poll()
1879 if (chan->in_fifo && !chan->in_synchronous && in xillyusb_poll()
1880 (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok)) in xillyusb_poll()
1883 if (chan->out_ep && in xillyusb_poll()
1884 (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size)) in xillyusb_poll()
1887 if (chan->xdev->error) in xillyusb_poll()
1906 struct usb_device *udev = xdev->udev; in xillyusb_setup_base_eps()
1911 return -ENODEV; in xillyusb_setup_base_eps()
1913 xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT, in xillyusb_setup_base_eps()
1915 if (!xdev->msg_ep) in xillyusb_setup_base_eps()
1916 return -ENOMEM; in xillyusb_setup_base_eps()
1918 if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */ in xillyusb_setup_base_eps()
1921 xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */ in xillyusb_setup_base_eps()
1923 xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN, in xillyusb_setup_base_eps()
1925 if (!xdev->in_ep) in xillyusb_setup_base_eps()
1928 try_queue_bulk_in(xdev->in_ep); in xillyusb_setup_base_eps()
1933 endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */ in xillyusb_setup_base_eps()
1934 xdev->msg_ep = NULL; in xillyusb_setup_base_eps()
1935 return -ENOMEM; in xillyusb_setup_base_eps()
1942 struct usb_device *udev = xdev->udev; in setup_channels()
1948 return -ENOMEM; in setup_channels()
1956 chan->xdev = xdev; in setup_channels()
1957 mutex_init(&chan->in_mutex); in setup_channels()
1958 mutex_init(&chan->out_mutex); in setup_channels()
1959 mutex_init(&chan->lock); in setup_channels()
1960 init_waitqueue_head(&chan->flushq); in setup_channels()
1962 chan->chan_idx = i; in setup_channels()
1965 chan->readable = 1; in setup_channels()
1966 chan->in_synchronous = !!(in_desc & 0x40); in setup_channels()
1967 chan->in_seekable = !!(in_desc & 0x20); in setup_channels()
1968 chan->in_log2_element_size = in_desc & 0x0f; in setup_channels()
1969 chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16; in setup_channels()
1973 * A downstream channel should never exist above index 13, in setup_channels()
1981 dev_err(xdev->dev, in setup_channels()
1985 return -ENODEV; in setup_channels()
1988 chan->writable = 1; in setup_channels()
1989 chan->out_synchronous = !!(out_desc & 0x40); in setup_channels()
1990 chan->out_seekable = !!(out_desc & 0x20); in setup_channels()
1991 chan->out_log2_element_size = out_desc & 0x0f; in setup_channels()
1992 chan->out_log2_fifo_size = in setup_channels()
1997 xdev->channels = new_channels; in setup_channels()
2015 dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n"); in xillyusb_discovery()
2019 /* Phase I: Set up one fake upstream channel and obtain IDT */ in xillyusb_discovery()
2035 chan = xdev->channels; in xillyusb_discovery()
2037 chan->in_fifo = &idt_fifo; in xillyusb_discovery()
2038 chan->read_data_ok = 1; in xillyusb_discovery()
2040 xdev->num_channels = 1; in xillyusb_discovery()
2045 dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n"); in xillyusb_discovery()
2050 !chan->read_data_ok, in xillyusb_discovery()
2053 if (xdev->error) { in xillyusb_discovery()
2054 rc = xdev->error; in xillyusb_discovery()
2059 rc = -EINTR; /* Interrupt on probe method? Interesting. */ in xillyusb_discovery()
2063 if (chan->read_data_ok) { in xillyusb_discovery()
2064 rc = -ETIMEDOUT; in xillyusb_discovery()
2065 dev_err(&interface->dev, "No response from FPGA. Aborting.\n"); in xillyusb_discovery()
2073 rc = -ENOMEM; in xillyusb_discovery()
2080 dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n"); in xillyusb_discovery()
2081 rc = -ENODEV; in xillyusb_discovery()
2086 …dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an up… in xillyusb_discovery()
2088 rc = -ENODEV; in xillyusb_discovery()
2096 idt_len -= 4; /* Exclude CRC */ in xillyusb_discovery()
2099 dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n"); in xillyusb_discovery()
2100 rc = -ENODEV; in xillyusb_discovery()
2112 * work item to be running now. To be sure that xdev->channels in xillyusb_discovery()
2117 flush_workqueue(xdev->workq); in xillyusb_discovery()
2118 flush_work(&xdev->wakeup_workitem); in xillyusb_discovery()
2120 xdev->num_channels = num_channels; in xillyusb_discovery()
2125 rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops, in xillyusb_discovery()
2128 idt_len - names_offset, in xillyusb_discovery()
2154 return -ENOMEM; in xillyusb_probe()
2156 kref_init(&xdev->kref); in xillyusb_probe()
2157 mutex_init(&xdev->process_in_mutex); in xillyusb_probe()
2158 mutex_init(&xdev->msg_mutex); in xillyusb_probe()
2160 xdev->udev = usb_get_dev(interface_to_usbdev(interface)); in xillyusb_probe()
2161 xdev->dev = &interface->dev; in xillyusb_probe()
2162 xdev->error = 0; in xillyusb_probe()
2163 spin_lock_init(&xdev->error_lock); in xillyusb_probe()
2164 xdev->in_counter = 0; in xillyusb_probe()
2165 xdev->in_bytes_left = 0; in xillyusb_probe()
2166 xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0); in xillyusb_probe()
2168 if (!xdev->workq) { in xillyusb_probe()
2169 dev_err(&interface->dev, "Failed to allocate work queue\n"); in xillyusb_probe()
2170 rc = -ENOMEM; in xillyusb_probe()
2174 INIT_WORK(&xdev->wakeup_workitem, wakeup_all); in xillyusb_probe()
2189 endpoint_quiesce(xdev->in_ep); in xillyusb_probe()
2190 endpoint_quiesce(xdev->msg_ep); in xillyusb_probe()
2194 kref_put(&xdev->kref, cleanup_dev); in xillyusb_probe()
2201 struct xillyusb_endpoint *msg_ep = xdev->msg_ep; in xillyusb_disconnect()
2202 struct xillyfifo *fifo = &msg_ep->fifo; in xillyusb_disconnect() local
2206 xillybus_cleanup_chrdev(xdev, &interface->dev); in xillyusb_disconnect()
2213 msg_ep->wake_on_drain = true; in xillyusb_disconnect()
2218 * a global device error with xdev->error, if such error didn't in xillyusb_disconnect()
2223 rc = wait_event_interruptible_timeout(fifo->waitq, in xillyusb_disconnect()
2224 msg_ep->drained || xdev->error, in xillyusb_disconnect()
2228 dev_err(&interface->dev, in xillyusb_disconnect()
2231 report_io_error(xdev, -ENODEV); /* Discourage further activity */ in xillyusb_disconnect()
2240 for (i = 0; i < xdev->num_channels; i++) { in xillyusb_disconnect()
2241 struct xillyusb_channel *chan = &xdev->channels[i]; in xillyusb_disconnect()
2244 * Lock taken to prevent chan->out_ep from changing. It also in xillyusb_disconnect()
2246 * xdev->dev after being nullified below. in xillyusb_disconnect()
2248 mutex_lock(&chan->lock); in xillyusb_disconnect()
2249 if (chan->out_ep) in xillyusb_disconnect()
2250 endpoint_quiesce(chan->out_ep); in xillyusb_disconnect()
2251 mutex_unlock(&chan->lock); in xillyusb_disconnect()
2254 endpoint_quiesce(xdev->in_ep); in xillyusb_disconnect()
2255 endpoint_quiesce(xdev->msg_ep); in xillyusb_disconnect()
2259 xdev->dev = NULL; in xillyusb_disconnect()
2262 kref_put(&xdev->kref, cleanup_dev); in xillyusb_disconnect()
2280 return -ENOMEM; in xillyusb_init()
2283 fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT; in xillyusb_init()