8  * a pipe between a hardware FIFO in the programmable logic and a device
76 	int fill; /* Number of bytes in the FIFO */
107 	struct xillyfifo fifo;  member
219  * calls to each on the same FIFO is not allowed) however it's OK to have
220  * threads calling each of the two functions once on the same FIFO, and
224 static int fifo_write(struct xillyfifo *fifo,  in fifo_write()  argument
231 	unsigned int writepos = fifo->writepos;  in fifo_write()
232 	unsigned int writebuf = fifo->writebuf;  in fifo_write()
236 	nmax = fifo->size - READ_ONCE(fifo->fill);  in fifo_write()
239 		unsigned int nrail = fifo->bufsize - writepos;  in fifo_write()
243 			spin_lock_irqsave(&fifo->lock, flags);  in fifo_write()
244 			fifo->fill += done;  in fifo_write()
245 			spin_unlock_irqrestore(&fifo->lock, flags);  in fifo_write()
247 			fifo->writepos = writepos;  in fifo_write()
248 			fifo->writebuf = writebuf;  in fifo_write()
256 		rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n);  in fifo_write()
267 		if (writepos == fifo->bufsize) {  in fifo_write()
271 			if (writebuf == fifo->bufnum)  in fifo_write()
277 static int fifo_read(struct xillyfifo *fifo,  in fifo_read()  argument
284 	unsigned int readpos = fifo->readpos;  in fifo_read()
285 	unsigned int readbuf = fifo->readbuf;  in fifo_read()
290 	 * The spinlock here is necessary, because otherwise fifo->fill  in fifo_read()
293 	 * visible on this thread at the time the updated fifo->fill was.  in fifo_read()
297 	spin_lock_irqsave(&fifo->lock, flags);  in fifo_read()
298 	fill = fifo->fill;  in fifo_read()
299 	spin_unlock_irqrestore(&fifo->lock, flags);  in fifo_read()
302 		unsigned int nrail = fifo->bufsize - readpos;  in fifo_read()
306 			spin_lock_irqsave(&fifo->lock, flags);  in fifo_read()
307 			fifo->fill -= done;  in fifo_read()
308 			spin_unlock_irqrestore(&fifo->lock, flags);  in fifo_read()
310 			fifo->readpos = readpos;  in fifo_read()
311 			fifo->readbuf = readbuf;  in fifo_read()
319 		rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n);  in fifo_read()
330 		if (readpos == fifo->bufsize) {  in fifo_read()
334 			if (readbuf == fifo->bufnum)  in fifo_read()
369 static int fifo_init(struct xillyfifo *fifo,  in fifo_init()  argument
384 		fifo->bufsize = 1 << log2_fifo_buf_size;  in fifo_init()
389 		fifo->bufsize = 1 << log2_size;  in fifo_init()
392 	fifo->bufnum = 1 << log2_bufnum;  in fifo_init()
393 	fifo->size = fifo->bufnum * fifo->bufsize;  in fifo_init()
394 	fifo->buf_order = buf_order;  in fifo_init()
396 	fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL);  in fifo_init()
398 	if (!fifo->mem)  in fifo_init()
401 	for (i = 0; i < fifo->bufnum; i++) {  in fifo_init()
402 		fifo->mem[i] = (void *)  in fifo_init()
405 		if (!fifo->mem[i])  in fifo_init()
409 	fifo->fill = 0;  in fifo_init()
410 	fifo->readpos = 0;  in fifo_init()
411 	fifo->readbuf = 0;  in fifo_init()
412 	fifo->writepos = 0;  in fifo_init()
413 	fifo->writebuf = 0;  in fifo_init()
414 	spin_lock_init(&fifo->lock);  in fifo_init()
415 	init_waitqueue_head(&fifo->waitq);  in fifo_init()
420 		free_pages((unsigned long)fifo->mem[i], buf_order);  in fifo_init()
422 	kfree(fifo->mem);  in fifo_init()
423 	fifo->mem = NULL;  in fifo_init()
433 static void fifo_mem_release(struct xillyfifo *fifo)  in fifo_mem_release()  argument
437 	if (!fifo->mem)  in fifo_mem_release()
440 	for (i = 0; i < fifo->bufnum; i++)  in fifo_mem_release()
441 		free_pages((unsigned long)fifo->mem[i], fifo->buf_order);  in fifo_mem_release()
443 	kfree(fifo->mem);  in fifo_mem_release()
463  * Note that endpoint_dealloc() also frees fifo memory (if allocated), even
471 	fifo_mem_release(&ep->fifo);  in endpoint_dealloc()
598 			wake_up_interruptible(&chan->out_ep->fifo.waitq);  in wakeup_all()
607 	wake_up_interruptible(&xdev->msg_ep->fifo.waitq);  in wakeup_all()
637 				  struct xillyfifo *fifo)  in safely_assign_in_fifo()  argument
640 	chan->in_fifo = fifo;  in safely_assign_in_fifo()
765 	struct xillyfifo *fifo = &ep->fifo;  in try_queue_bulk_out()  local
780 	fill = READ_ONCE(fifo->fill) & ep->fill_mask;  in try_queue_bulk_out()
789 		 * Race conditions might have the FIFO filled while the  in try_queue_bulk_out()
793 		 * shutting it down. Hence knowing that the FIFO appears  in try_queue_bulk_out()
823 		count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy);  in try_queue_bulk_out()
870 		wake_up_interruptible(&fifo->waitq);  in try_queue_bulk_out()
906 		 * A write memory barrier ensures that the FIFO's fill level  in process_in_opcode()
908 		 * the FIFO isn't missed by the consumer.  in process_in_opcode()
943 	struct xillyfifo *fifo;  in process_bulk_in()  local
1002 		fifo = chan->in_fifo;  in process_bulk_in()
1004 		if (unlikely(!fifo))  in process_bulk_in()
1007 		if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) {  in process_bulk_in()
1008 			dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n");  in process_bulk_in()
1012 		wake_up_interruptible(&fifo->waitq);  in process_bulk_in()
1072 	struct xillyfifo *fifo = &ep->fifo;  in xillyusb_send_opcode()  local
1091 	while (wait_event_interruptible(fifo->waitq,  in xillyusb_send_opcode()
1092 					fifo->fill <= (fifo->size - 8) ||  in xillyusb_send_opcode()
1101 	fifo_write(fifo, (void *)msg, 8, xilly_memcpy);  in xillyusb_send_opcode()
1174 	 * FIFO, it's not flushed, including the flush before closing, which  in flush_downstream()
1308 		rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size);  in xillyusb_open()
1436 	struct xillyfifo *fifo = chan->in_fifo;  in xillyusb_read()  local
1459 		rc = fifo_read(fifo, (__force void *)userbuf + bytes_done,  in xillyusb_read()
1478 		fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size;  in xillyusb_read()
1501 		 * checkpoint by at least an 8th of the FIFO's size, or if  in xillyusb_read()
1511 		    (leap > (fifo->size >> 3) ||  in xillyusb_read()
1527 		 * Reaching here means that the FIFO was empty when  in xillyusb_read()
1530 		 * that managed its way to the FIFO is lost.  in xillyusb_read()
1534 			/* Has data slipped into the FIFO since fifo_read()? */  in xillyusb_read()
1536 			if (READ_ONCE(fifo->fill))  in xillyusb_read()
1568 			 * fifo->waitq is awaken.  in xillyusb_read()
1573 				(fifo->waitq,  in xillyusb_read()
1574 				 fifo->fill || !chan->read_data_ok,  in xillyusb_read()
1584 				(fifo->waitq,  in xillyusb_read()
1585 				 fifo->fill || !chan->read_data_ok);  in xillyusb_read()
1595 	    !READ_ONCE(fifo->fill))  in xillyusb_read()
1647 	struct xillyfifo *fifo = &chan->out_ep->fifo;  in xillyusb_write()  local
1664 		rc = fifo_write(fifo, (__force void *)userbuf, count,  in xillyusb_write()
1676 		    (fifo->waitq,  in xillyusb_write()
1677 		     fifo->fill != fifo->size || xdev->error)) {  in xillyusb_write()
1855 		poll_wait(filp, &chan->out_ep->fifo.waitq, wait);  in xillyusb_poll()
1884 	    (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size))  in xillyusb_poll()
1918 	if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */  in xillyusb_setup_base_eps()
1933 	endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */  in xillyusb_setup_base_eps()
2202 	struct xillyfifo *fifo = &msg_ep->fifo;  in xillyusb_disconnect()  local
2223 	rc = wait_event_interruptible_timeout(fifo->waitq,  in xillyusb_disconnect()