Lines Matching +full:buffer +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2013-2015 Analog Devices Inc.
4 * Author: Lars-Peter Clausen <lars@metafoo.de>
18 #include <linux/iio/buffer-dma.h>
19 #include <linux/dma-buf.h>
20 #include <linux/dma-fence.h>
21 #include <linux/dma-mapping.h>
25 * For DMA buffers the storage is sub-divided into so called blocks. Each block
26 * has its own memory buffer. The size of the block is the granularity at which
73 * the actual number of bytes in the buffer. Typically this will be equal to the
74 * size of the block, but if the DMA hardware has certain alignment requirements
75 * for the transfer length it might choose to use less than the full size. In
81 * perform a DMA transfer for the block, e.g. because the buffer was disabled
86 * It will be called when the buffer is disabled and can be used to cancel
101 struct iio_dma_buffer_queue *queue = block->queue;
103 WARN_ON(block->fileio && block->state != IIO_BLOCK_STATE_DEAD);
105 if (block->fileio) {
106 dma_free_coherent(queue->dev, PAGE_ALIGN(block->size),
107 block->vaddr, block->phys_addr);
109 atomic_dec(&queue->num_dmabufs);
112 iio_buffer_put(&queue->buffer);
118 kref_get(&block->kref);
123 kref_put(&block->kref, iio_buffer_block_release);
143 iio_buffer_block_release(&block->kref);
155 list_add_tail(&block->head, &iio_dma_buffer_dead_blocks);
166 kref_put(&block->kref, iio_buffer_block_release_atomic);
171 return container_of(buf, struct iio_dma_buffer_queue, buffer);
175 struct iio_dma_buffer_queue *queue, size_t size, bool fileio)
184 block->vaddr = dma_alloc_coherent(queue->dev, PAGE_ALIGN(size),
185 &block->phys_addr, GFP_KERNEL);
186 if (!block->vaddr) {
192 block->fileio = fileio;
193 block->size = size;
194 block->state = IIO_BLOCK_STATE_DONE;
195 block->queue = queue;
196 INIT_LIST_HEAD(&block->head);
197 kref_init(&block->kref);
199 iio_buffer_get(&queue->buffer);
202 atomic_inc(&queue->num_dmabufs);
209 if (block->state != IIO_BLOCK_STATE_DEAD)
210 block->state = IIO_BLOCK_STATE_DONE;
217 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN)
222 wake_up_interruptible_poll(&queue->buffer.pollq, flags);
226 * iio_dma_buffer_block_done() - Indicate that a block has been completed
234 struct iio_dma_buffer_queue *queue = block->queue;
240 spin_lock_irqsave(&queue->list_lock, flags);
242 spin_unlock_irqrestore(&queue->list_lock, flags);
244 if (!block->fileio)
245 iio_buffer_signal_dmabuf_done(block->fence, 0);
254 * iio_dma_buffer_block_list_abort() - Indicate that a list block has been
272 spin_lock_irqsave(&queue->list_lock, flags);
274 list_del(&block->head);
275 block->bytes_used = 0;
278 if (!block->fileio)
279 iio_buffer_signal_dmabuf_done(block->fence, -EINTR);
282 spin_unlock_irqrestore(&queue->list_lock, flags);
284 if (queue->fileio.enabled)
285 queue->fileio.enabled = false;
295 * If the core owns the block it can be re-used. This should be the
296 * default case when enabling the buffer, unless the DMA controller does
299 switch (block->state) {
311 * Note that queue->num_dmabufs cannot increase while the queue is
315 return queue->fileio.enabled || !atomic_read(&queue->num_dmabufs);
319 * iio_dma_buffer_request_update() - DMA buffer request_update callback
320 * @buffer: The buffer which to request an update
325 int iio_dma_buffer_request_update(struct iio_buffer *buffer)
327 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
330 size_t size;
335 * Split the buffer into two even parts. This is used as a double
339 size = DIV_ROUND_UP(queue->buffer.bytes_per_datum *
340 queue->buffer.length, 2);
342 mutex_lock(&queue->lock);
344 queue->fileio.enabled = iio_dma_buffer_can_use_fileio(queue);
347 if (!queue->fileio.enabled)
351 if (PAGE_ALIGN(queue->fileio.block_size) == PAGE_ALIGN(size))
354 queue->fileio.block_size = size;
355 queue->fileio.active_block = NULL;
357 spin_lock_irq(&queue->list_lock);
358 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
359 block = queue->fileio.blocks[i];
361 /* If we can't re-use it free it */
363 block->state = IIO_BLOCK_STATE_DEAD;
371 spin_unlock_irq(&queue->list_lock);
373 INIT_LIST_HEAD(&queue->incoming);
375 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
376 if (queue->fileio.blocks[i]) {
377 block = queue->fileio.blocks[i];
378 if (block->state == IIO_BLOCK_STATE_DEAD) {
383 block->size = size;
390 block = iio_dma_buffer_alloc_block(queue, size, true);
392 ret = -ENOMEM;
395 queue->fileio.blocks[i] = block;
399 * block->bytes_used may have been modified previously, e.g. by
403 block->bytes_used = block->size;
406 * If it's an input buffer, mark the block as queued, and
410 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) {
411 block->state = IIO_BLOCK_STATE_QUEUED;
412 list_add_tail(&block->head, &queue->incoming);
414 block->state = IIO_BLOCK_STATE_DONE;
419 mutex_unlock(&queue->lock);
429 spin_lock_irq(&queue->list_lock);
430 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
431 if (!queue->fileio.blocks[i])
433 queue->fileio.blocks[i]->state = IIO_BLOCK_STATE_DEAD;
435 spin_unlock_irq(&queue->list_lock);
437 INIT_LIST_HEAD(&queue->incoming);
439 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
440 if (!queue->fileio.blocks[i])
442 iio_buffer_block_put(queue->fileio.blocks[i]);
443 queue->fileio.blocks[i] = NULL;
445 queue->fileio.active_block = NULL;
458 if (!queue->ops)
461 block->state = IIO_BLOCK_STATE_ACTIVE;
464 ret = queue->ops->submit(queue, block);
466 if (!block->fileio)
467 iio_buffer_signal_dmabuf_done(block->fence, ret);
471 * other then wait for the buffer to be disabled and re-enabled
476 * notify consumers that something went wrong and the buffer
484 * iio_dma_buffer_enable() - Enable DMA buffer
485 * @buffer: IIO buffer to enable
486 * @indio_dev: IIO device the buffer is attached to
488 * Needs to be called when the device that the buffer is attached to starts
493 int iio_dma_buffer_enable(struct iio_buffer *buffer,
496 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
499 mutex_lock(&queue->lock);
500 queue->active = true;
501 list_for_each_entry_safe(block, _block, &queue->incoming, head) {
502 list_del(&block->head);
505 mutex_unlock(&queue->lock);
512 * iio_dma_buffer_disable() - Disable DMA buffer
513 * @buffer: IIO DMA buffer to disable
514 * @indio_dev: IIO device the buffer is attached to
516 * Needs to be called when the device that the buffer is attached to stops
519 int iio_dma_buffer_disable(struct iio_buffer *buffer,
522 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
524 mutex_lock(&queue->lock);
525 queue->active = false;
527 if (queue->ops && queue->ops->abort)
528 queue->ops->abort(queue);
529 mutex_unlock(&queue->lock);
538 if (block->state == IIO_BLOCK_STATE_DEAD) {
540 } else if (queue->active) {
543 block->state = IIO_BLOCK_STATE_QUEUED;
544 list_add_tail(&block->head, &queue->incoming);
554 spin_lock_irq(&queue->list_lock);
556 idx = queue->fileio.next_dequeue;
557 block = queue->fileio.blocks[idx];
559 if (block->state == IIO_BLOCK_STATE_DONE) {
560 idx = (idx + 1) % ARRAY_SIZE(queue->fileio.blocks);
561 queue->fileio.next_dequeue = idx;
566 spin_unlock_irq(&queue->list_lock);
571 static int iio_dma_buffer_io(struct iio_buffer *buffer, size_t n,
574 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
579 if (n < buffer->bytes_per_datum)
580 return -EINVAL;
582 mutex_lock(&queue->lock);
584 if (!queue->fileio.active_block) {
590 queue->fileio.pos = 0;
591 queue->fileio.active_block = block;
593 block = queue->fileio.active_block;
596 n = rounddown(n, buffer->bytes_per_datum);
597 if (n > block->bytes_used - queue->fileio.pos)
598 n = block->bytes_used - queue->fileio.pos;
599 addr = block->vaddr + queue->fileio.pos;
606 ret = -EFAULT;
610 queue->fileio.pos += n;
612 if (queue->fileio.pos == block->bytes_used) {
613 queue->fileio.active_block = NULL;
620 mutex_unlock(&queue->lock);
626 * iio_dma_buffer_read() - DMA buffer read callback
627 * @buffer: Buffer to read from
629 * @user_buffer: Userspace buffer to copy the data to
634 int iio_dma_buffer_read(struct iio_buffer *buffer, size_t n,
637 return iio_dma_buffer_io(buffer, n, user_buffer, false);
642 * iio_dma_buffer_write() - DMA buffer write callback
643 * @buffer: Buffer to write to
645 * @user_buffer: Userspace buffer to copy the data from
650 int iio_dma_buffer_write(struct iio_buffer *buffer, size_t n,
653 return iio_dma_buffer_io(buffer, n,
659 * iio_dma_buffer_usage() - DMA buffer data_available and
661 * @buf: Buffer to check for data availability
674 * For counting the available bytes we'll use the size of the block not
680 mutex_lock(&queue->lock);
681 if (queue->fileio.active_block)
682 data_available += queue->fileio.active_block->size;
684 spin_lock_irq(&queue->list_lock);
686 for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
687 block = queue->fileio.blocks[i];
689 if (block != queue->fileio.active_block
690 && block->state == IIO_BLOCK_STATE_DONE)
691 data_available += block->size;
694 spin_unlock_irq(&queue->list_lock);
695 mutex_unlock(&queue->lock);
702 iio_dma_buffer_attach_dmabuf(struct iio_buffer *buffer,
705 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
708 guard(mutex)(&queue->lock);
711 * If the buffer is enabled and in fileio mode new blocks can't be
714 if (queue->fileio.enabled)
715 return ERR_PTR(-EBUSY);
717 block = iio_dma_buffer_alloc_block(queue, attach->dmabuf->size, false);
719 return ERR_PTR(-ENOMEM);
728 void iio_dma_buffer_detach_dmabuf(struct iio_buffer *buffer,
731 block->state = IIO_BLOCK_STATE_DEAD;
738 struct iio_dma_buffer_queue *queue = block->queue;
741 if (queue->fileio.enabled)
742 return -EBUSY;
744 switch (block->state) {
746 return -EPERM;
749 return -EBUSY;
757 int iio_dma_buffer_enqueue_dmabuf(struct iio_buffer *buffer,
761 size_t size, bool cyclic)
763 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
767 WARN_ON(!mutex_is_locked(&queue->lock));
775 block->bytes_used = size;
776 block->cyclic = cyclic;
777 block->sg_table = sgt;
778 block->fence = fence;
789 void iio_dma_buffer_lock_queue(struct iio_buffer *buffer)
791 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
793 mutex_lock(&queue->lock);
797 void iio_dma_buffer_unlock_queue(struct iio_buffer *buffer)
799 struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
801 mutex_unlock(&queue->lock);
806 * iio_dma_buffer_set_bytes_per_datum() - DMA buffer set_bytes_per_datum callback
807 * @buffer: Buffer to set the bytes-per-datum for
808 * @bpd: The new bytes-per-datum value
813 int iio_dma_buffer_set_bytes_per_datum(struct iio_buffer *buffer, size_t bpd)
815 buffer->bytes_per_datum = bpd;
822 * iio_dma_buffer_set_length - DMA buffer set_length callback
823 * @buffer: Buffer to set the length for
824 * @length: The new buffer length
829 int iio_dma_buffer_set_length(struct iio_buffer *buffer, unsigned int length)
834 buffer->length = length;
835 buffer->watermark = length / 2;
842 * iio_dma_buffer_init() - Initialize DMA buffer queue
843 * @queue: Buffer to initialize
845 * @ops: DMA buffer queue callback operations
854 iio_buffer_init(&queue->buffer);
855 queue->buffer.length = PAGE_SIZE;
856 queue->buffer.watermark = queue->buffer.length / 2;
857 queue->dev = dev;
858 queue->ops = ops;
860 INIT_LIST_HEAD(&queue->incoming);
862 mutex_init(&queue->lock);
863 spin_lock_init(&queue->list_lock);
870 * iio_dma_buffer_exit() - Cleanup DMA buffer queue
871 * @queue: Buffer to cleanup
874 * associated with the buffer and are accessed inside the callback operations.
878 mutex_lock(&queue->lock);
881 queue->ops = NULL;
883 mutex_unlock(&queue->lock);
888 * iio_dma_buffer_release() - Release final buffer resources
889 * @queue: Buffer to release
893 * the memory associated with the buffer.
897 mutex_destroy(&queue->lock);
901 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
902 MODULE_DESCRIPTION("DMA buffer for the IIO framework");