1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright 2014-2015 Analog Devices Inc.
4 * Author: Lars-Peter Clausen <lars@metafoo.de>
5 */
6
7 #include <linux/slab.h>
8 #include <linux/kernel.h>
9 #include <linux/cleanup.h>
10 #include <linux/dmaengine.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/spinlock.h>
13 #include <linux/err.h>
14 #include <linux/module.h>
15
16 #include <linux/iio/iio.h>
17 #include <linux/iio/sysfs.h>
18 #include <linux/iio/buffer.h>
19 #include <linux/iio/buffer_impl.h>
20 #include <linux/iio/buffer-dma.h>
21 #include <linux/iio/buffer-dmaengine.h>
22
23 /*
24 * The IIO DMAengine buffer combines the generic IIO DMA buffer infrastructure
25 * with the DMAengine framework. The generic IIO DMA buffer infrastructure is
26 * used to manage the buffer memory and implement the IIO buffer operations
27 * while the DMAengine framework is used to perform the DMA transfers. Combined
28 * this results in a device independent fully functional DMA buffer
29 * implementation that can be used by device drivers for peripherals which are
30 * connected to a DMA controller which has a DMAengine driver implementation.
31 */
32
33 struct dmaengine_buffer {
34 struct iio_dma_buffer_queue queue;
35
36 struct dma_chan *chan;
37 struct list_head active;
38
39 size_t align;
40 size_t max_size;
41 };
42
iio_buffer_to_dmaengine_buffer(struct iio_buffer * buffer)43 static struct dmaengine_buffer *iio_buffer_to_dmaengine_buffer(struct iio_buffer *buffer)
44 {
45 return container_of(buffer, struct dmaengine_buffer, queue.buffer);
46 }
47
iio_dmaengine_buffer_block_done(void * data,const struct dmaengine_result * result)48 static void iio_dmaengine_buffer_block_done(void *data,
49 const struct dmaengine_result *result)
50 {
51 struct iio_dma_buffer_block *block = data;
52
53 scoped_guard(spinlock_irqsave, &block->queue->list_lock)
54 list_del(&block->head);
55 block->bytes_used -= result->residue;
56 iio_dma_buffer_block_done(block);
57 }
58
59 /*
60 * Cyclic transfers run until abort. Cap active cyclic blocks to one until there
61 * is a use case for more.
62 */
iio_dmaengine_buffer_has_active_cyclic(struct dmaengine_buffer * dmaengine_buffer)63 static bool iio_dmaengine_buffer_has_active_cyclic(struct dmaengine_buffer *dmaengine_buffer)
64 {
65 struct iio_dma_buffer_block *block;
66
67 guard(spinlock_irqsave)(&dmaengine_buffer->queue.list_lock);
68 list_for_each_entry(block, &dmaengine_buffer->active, head) {
69 if (block->cyclic)
70 return true;
71 }
72
73 return false;
74 }
75
iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue * queue,struct iio_dma_buffer_block * block)76 static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue,
77 struct iio_dma_buffer_block *block)
78 {
79 struct dmaengine_buffer *dmaengine_buffer =
80 iio_buffer_to_dmaengine_buffer(&queue->buffer);
81 struct dma_async_tx_descriptor *desc;
82 enum dma_transfer_direction dma_dir;
83 struct scatterlist *sgl;
84 struct dma_vec *vecs;
85 size_t max_size;
86 dma_cookie_t cookie;
87 size_t len_total;
88 unsigned int i;
89 int nents;
90
91 max_size = min(block->size, dmaengine_buffer->max_size);
92 max_size = round_down(max_size, dmaengine_buffer->align);
93
94 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN)
95 dma_dir = DMA_DEV_TO_MEM;
96 else
97 dma_dir = DMA_MEM_TO_DEV;
98
99 if (block->cyclic && iio_dmaengine_buffer_has_active_cyclic(dmaengine_buffer)) {
100 dev_err(queue->dev, "cyclic DMA transfer already active\n");
101 return -EBUSY;
102 }
103
104 if (block->sg_table) {
105 unsigned long flags;
106
107 sgl = block->sg_table->sgl;
108 nents = sg_nents_for_len(sgl, block->bytes_used);
109 if (nents < 0)
110 return nents;
111
112 vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC);
113 if (!vecs)
114 return -ENOMEM;
115
116 len_total = block->bytes_used;
117
118 for (i = 0; i < nents; i++) {
119 vecs[i].addr = sg_dma_address(sgl);
120 vecs[i].len = min(sg_dma_len(sgl), len_total);
121 len_total -= vecs[i].len;
122
123 sgl = sg_next(sgl);
124 }
125
126 if (block->cyclic)
127 flags = DMA_PREP_REPEAT;
128 else
129 flags = DMA_PREP_INTERRUPT;
130
131 /*
132 * A new transfer may need to end an already active cyclic transfer
133 * before it can run, so always set the EOT flag.
134 */
135 flags |= DMA_PREP_LOAD_EOT;
136 desc = dmaengine_prep_peripheral_dma_vec(dmaengine_buffer->chan,
137 vecs, nents, dma_dir,
138 flags);
139 kfree(vecs);
140 } else {
141 max_size = min(block->size, dmaengine_buffer->max_size);
142 max_size = round_down(max_size, dmaengine_buffer->align);
143
144 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN)
145 block->bytes_used = max_size;
146
147 if (!block->bytes_used || block->bytes_used > max_size)
148 return -EINVAL;
149
150 desc = dmaengine_prep_slave_single(dmaengine_buffer->chan,
151 block->phys_addr,
152 block->bytes_used,
153 dma_dir,
154 DMA_PREP_INTERRUPT);
155 }
156 if (!desc)
157 return -ENOMEM;
158
159 if (!block->cyclic) {
160 desc->callback_result = iio_dmaengine_buffer_block_done;
161 desc->callback_param = block;
162 }
163
164 cookie = dmaengine_submit(desc);
165 if (dma_submit_error(cookie))
166 return dma_submit_error(cookie);
167
168 scoped_guard(spinlock_irq, &dmaengine_buffer->queue.list_lock)
169 list_add_tail(&block->head, &dmaengine_buffer->active);
170
171 dma_async_issue_pending(dmaengine_buffer->chan);
172
173 return 0;
174 }
175
iio_dmaengine_buffer_abort(struct iio_dma_buffer_queue * queue)176 static void iio_dmaengine_buffer_abort(struct iio_dma_buffer_queue *queue)
177 {
178 struct dmaengine_buffer *dmaengine_buffer =
179 iio_buffer_to_dmaengine_buffer(&queue->buffer);
180
181 dmaengine_terminate_sync(dmaengine_buffer->chan);
182 iio_dma_buffer_block_list_abort(queue, &dmaengine_buffer->active);
183 }
184
iio_dmaengine_buffer_release(struct iio_buffer * buf)185 static void iio_dmaengine_buffer_release(struct iio_buffer *buf)
186 {
187 struct dmaengine_buffer *dmaengine_buffer =
188 iio_buffer_to_dmaengine_buffer(buf);
189
190 iio_dma_buffer_release(&dmaengine_buffer->queue);
191 kfree(dmaengine_buffer);
192 }
193
194 static const struct iio_buffer_access_funcs iio_dmaengine_buffer_ops = {
195 .read = iio_dma_buffer_read,
196 .write = iio_dma_buffer_write,
197 .set_bytes_per_datum = iio_dma_buffer_set_bytes_per_datum,
198 .set_length = iio_dma_buffer_set_length,
199 .request_update = iio_dma_buffer_request_update,
200 .enable = iio_dma_buffer_enable,
201 .disable = iio_dma_buffer_disable,
202 .data_available = iio_dma_buffer_usage,
203 .space_available = iio_dma_buffer_usage,
204 .release = iio_dmaengine_buffer_release,
205
206 .enqueue_dmabuf = iio_dma_buffer_enqueue_dmabuf,
207 .attach_dmabuf = iio_dma_buffer_attach_dmabuf,
208 .detach_dmabuf = iio_dma_buffer_detach_dmabuf,
209
210 .lock_queue = iio_dma_buffer_lock_queue,
211 .unlock_queue = iio_dma_buffer_unlock_queue,
212
213 .get_dma_dev = iio_dma_buffer_get_dma_dev,
214
215 .modes = INDIO_BUFFER_HARDWARE,
216 .flags = INDIO_BUFFER_FLAG_FIXED_WATERMARK,
217 };
218
219 static const struct iio_dma_buffer_ops iio_dmaengine_default_ops = {
220 .submit = iio_dmaengine_buffer_submit_block,
221 .abort = iio_dmaengine_buffer_abort,
222 };
223
iio_dmaengine_buffer_get_length_align(struct device * dev,struct device_attribute * attr,char * buf)224 static ssize_t iio_dmaengine_buffer_get_length_align(struct device *dev,
225 struct device_attribute *attr, char *buf)
226 {
227 struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
228 struct dmaengine_buffer *dmaengine_buffer =
229 iio_buffer_to_dmaengine_buffer(buffer);
230
231 return sysfs_emit(buf, "%zu\n", dmaengine_buffer->align);
232 }
233
234 static IIO_DEVICE_ATTR(length_align_bytes, 0444,
235 iio_dmaengine_buffer_get_length_align, NULL, 0);
236
237 static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = {
238 &iio_dev_attr_length_align_bytes,
239 NULL,
240 };
241
242 /**
243 * iio_dmaengine_buffer_alloc() - Allocate new buffer which uses DMAengine
244 * @chan: DMA channel.
245 *
246 * This allocates a new IIO buffer which internally uses the DMAengine framework
247 * to perform its transfers.
248 *
249 * Once done using the buffer iio_dmaengine_buffer_free() should be used to
250 * release it.
251 */
iio_dmaengine_buffer_alloc(struct dma_chan * chan)252 static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan)
253 {
254 struct dmaengine_buffer *dmaengine_buffer;
255 unsigned int width, src_width, dest_width;
256 struct dma_slave_caps caps;
257 int ret;
258
259 ret = dma_get_slave_caps(chan, &caps);
260 if (ret < 0)
261 return ERR_PTR(ret);
262
263 dmaengine_buffer = kzalloc_obj(*dmaengine_buffer);
264 if (!dmaengine_buffer)
265 return ERR_PTR(-ENOMEM);
266
267 /* Needs to be aligned to the maximum of the minimums */
268 if (caps.src_addr_widths)
269 src_width = __ffs(caps.src_addr_widths);
270 else
271 src_width = 1;
272 if (caps.dst_addr_widths)
273 dest_width = __ffs(caps.dst_addr_widths);
274 else
275 dest_width = 1;
276 width = max(src_width, dest_width);
277
278 INIT_LIST_HEAD(&dmaengine_buffer->active);
279 dmaengine_buffer->chan = chan;
280 dmaengine_buffer->align = width;
281 dmaengine_buffer->max_size = dma_get_max_seg_size(chan->device->dev);
282
283 iio_dma_buffer_init(&dmaengine_buffer->queue, chan->device->dev,
284 &iio_dmaengine_default_ops);
285
286 dmaengine_buffer->queue.buffer.attrs = iio_dmaengine_buffer_attrs;
287 dmaengine_buffer->queue.buffer.access = &iio_dmaengine_buffer_ops;
288
289 return &dmaengine_buffer->queue.buffer;
290 }
291
292 /**
293 * iio_dmaengine_buffer_free() - Free dmaengine buffer
294 * @buffer: Buffer to free
295 *
296 * Frees a buffer previously allocated with iio_dmaengine_buffer_alloc().
297 */
iio_dmaengine_buffer_free(struct iio_buffer * buffer)298 static void iio_dmaengine_buffer_free(struct iio_buffer *buffer)
299 {
300 struct dmaengine_buffer *dmaengine_buffer =
301 iio_buffer_to_dmaengine_buffer(buffer);
302
303 iio_dma_buffer_exit(&dmaengine_buffer->queue);
304 iio_buffer_put(buffer);
305 }
306
307 /**
308 * iio_dmaengine_buffer_teardown() - Releases DMA channel and frees buffer
309 * @buffer: Buffer to free
310 *
311 * Releases the DMA channel and frees the buffer previously setup with
312 * iio_dmaengine_buffer_setup_ext().
313 */
iio_dmaengine_buffer_teardown(struct iio_buffer * buffer)314 void iio_dmaengine_buffer_teardown(struct iio_buffer *buffer)
315 {
316 struct dmaengine_buffer *dmaengine_buffer =
317 iio_buffer_to_dmaengine_buffer(buffer);
318 struct dma_chan *chan = dmaengine_buffer->chan;
319
320 iio_dmaengine_buffer_free(buffer);
321 dma_release_channel(chan);
322 }
323 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_teardown, "IIO_DMAENGINE_BUFFER");
324
325 static struct iio_buffer
__iio_dmaengine_buffer_setup_ext(struct iio_dev * indio_dev,struct dma_chan * chan,enum iio_buffer_direction dir)326 *__iio_dmaengine_buffer_setup_ext(struct iio_dev *indio_dev,
327 struct dma_chan *chan,
328 enum iio_buffer_direction dir)
329 {
330 struct iio_buffer *buffer;
331 int ret;
332
333 buffer = iio_dmaengine_buffer_alloc(chan);
334 if (IS_ERR(buffer))
335 return ERR_CAST(buffer);
336
337 indio_dev->modes |= INDIO_BUFFER_HARDWARE;
338
339 buffer->direction = dir;
340
341 ret = iio_device_attach_buffer(indio_dev, buffer);
342 if (ret) {
343 iio_dmaengine_buffer_free(buffer);
344 return ERR_PTR(ret);
345 }
346
347 return buffer;
348 }
349
350 /**
351 * iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device
352 * @dev: DMA channel consumer device
353 * @indio_dev: IIO device to which to attach this buffer.
354 * @channel: DMA channel name, typically "rx".
355 * @dir: Direction of buffer (in or out)
356 *
357 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc()
358 * and attaches it to an IIO device with iio_device_attach_buffer().
359 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the
360 * IIO device.
361 *
362 * Once done using the buffer iio_dmaengine_buffer_teardown() should be used to
363 * release it.
364 */
iio_dmaengine_buffer_setup_ext(struct device * dev,struct iio_dev * indio_dev,const char * channel,enum iio_buffer_direction dir)365 struct iio_buffer *iio_dmaengine_buffer_setup_ext(struct device *dev,
366 struct iio_dev *indio_dev,
367 const char *channel,
368 enum iio_buffer_direction dir)
369 {
370 struct dma_chan *chan;
371 struct iio_buffer *buffer;
372
373 chan = dma_request_chan(dev, channel);
374 if (IS_ERR(chan))
375 return ERR_CAST(chan);
376
377 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir);
378 if (IS_ERR(buffer))
379 dma_release_channel(chan);
380
381 return buffer;
382 }
383 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER");
384
devm_iio_dmaengine_buffer_teardown(void * buffer)385 static void devm_iio_dmaengine_buffer_teardown(void *buffer)
386 {
387 iio_dmaengine_buffer_teardown(buffer);
388 }
389
390 /**
391 * devm_iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device
392 * @dev: Device for devm ownership and DMA channel consumer device
393 * @indio_dev: IIO device to which to attach this buffer.
394 * @channel: DMA channel name, typically "rx".
395 * @dir: Direction of buffer (in or out)
396 *
397 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc()
398 * and attaches it to an IIO device with iio_device_attach_buffer().
399 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the
400 * IIO device.
401 */
devm_iio_dmaengine_buffer_setup_ext(struct device * dev,struct iio_dev * indio_dev,const char * channel,enum iio_buffer_direction dir)402 int devm_iio_dmaengine_buffer_setup_ext(struct device *dev,
403 struct iio_dev *indio_dev,
404 const char *channel,
405 enum iio_buffer_direction dir)
406 {
407 struct iio_buffer *buffer;
408
409 buffer = iio_dmaengine_buffer_setup_ext(dev, indio_dev, channel, dir);
410 if (IS_ERR(buffer))
411 return PTR_ERR(buffer);
412
413 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_teardown,
414 buffer);
415 }
416 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER");
417
devm_iio_dmaengine_buffer_free(void * buffer)418 static void devm_iio_dmaengine_buffer_free(void *buffer)
419 {
420 iio_dmaengine_buffer_free(buffer);
421 }
422
423 /**
424 * devm_iio_dmaengine_buffer_setup_with_handle() - Setup a DMA buffer for an
425 * IIO device
426 * @dev: Device for devm ownership
427 * @indio_dev: IIO device to which to attach this buffer.
428 * @chan: DMA channel
429 * @dir: Direction of buffer (in or out)
430 *
431 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc()
432 * and attaches it to an IIO device with iio_device_attach_buffer().
433 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the
434 * IIO device.
435 *
436 * This is the same as devm_iio_dmaengine_buffer_setup_ext() except that the
437 * caller manages requesting and releasing the DMA channel handle.
438 */
devm_iio_dmaengine_buffer_setup_with_handle(struct device * dev,struct iio_dev * indio_dev,struct dma_chan * chan,enum iio_buffer_direction dir)439 int devm_iio_dmaengine_buffer_setup_with_handle(struct device *dev,
440 struct iio_dev *indio_dev,
441 struct dma_chan *chan,
442 enum iio_buffer_direction dir)
443 {
444 struct iio_buffer *buffer;
445
446 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir);
447 if (IS_ERR(buffer))
448 return PTR_ERR(buffer);
449
450 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_free,
451 buffer);
452 }
453 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_with_handle,
454 "IIO_DMAENGINE_BUFFER");
455
456 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
457 MODULE_DESCRIPTION("DMA buffer for the IIO framework");
458 MODULE_LICENSE("GPL");
459 MODULE_IMPORT_NS("IIO_DMA_BUFFER");
460