1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2014-2015 Analog Devices Inc. 4 * Author: Lars-Peter Clausen <lars@metafoo.de> 5 */ 6 7 #include <linux/slab.h> 8 #include <linux/kernel.h> 9 #include <linux/cleanup.h> 10 #include <linux/dmaengine.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/spinlock.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 16 #include <linux/iio/iio.h> 17 #include <linux/iio/sysfs.h> 18 #include <linux/iio/buffer.h> 19 #include <linux/iio/buffer_impl.h> 20 #include <linux/iio/buffer-dma.h> 21 #include <linux/iio/buffer-dmaengine.h> 22 23 /* 24 * The IIO DMAengine buffer combines the generic IIO DMA buffer infrastructure 25 * with the DMAengine framework. The generic IIO DMA buffer infrastructure is 26 * used to manage the buffer memory and implement the IIO buffer operations 27 * while the DMAengine framework is used to perform the DMA transfers. Combined 28 * this results in a device independent fully functional DMA buffer 29 * implementation that can be used by device drivers for peripherals which are 30 * connected to a DMA controller which has a DMAengine driver implementation. 31 */ 32 33 struct dmaengine_buffer { 34 struct iio_dma_buffer_queue queue; 35 36 struct dma_chan *chan; 37 struct list_head active; 38 39 size_t align; 40 size_t max_size; 41 }; 42 43 static struct dmaengine_buffer *iio_buffer_to_dmaengine_buffer(struct iio_buffer *buffer) 44 { 45 return container_of(buffer, struct dmaengine_buffer, queue.buffer); 46 } 47 48 static void iio_dmaengine_buffer_block_done(void *data, 49 const struct dmaengine_result *result) 50 { 51 struct iio_dma_buffer_block *block = data; 52 53 scoped_guard(spinlock_irqsave, &block->queue->list_lock) 54 list_del(&block->head); 55 block->bytes_used -= result->residue; 56 iio_dma_buffer_block_done(block); 57 } 58 59 /* 60 * Cyclic transfers run until abort. Cap active cyclic blocks to one until there 61 * is a use case for more. 62 */ 63 static bool iio_dmaengine_buffer_has_active_cyclic(struct dmaengine_buffer *dmaengine_buffer) 64 { 65 struct iio_dma_buffer_block *block; 66 67 guard(spinlock_irqsave)(&dmaengine_buffer->queue.list_lock); 68 list_for_each_entry(block, &dmaengine_buffer->active, head) { 69 if (block->cyclic) 70 return true; 71 } 72 73 return false; 74 } 75 76 static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue, 77 struct iio_dma_buffer_block *block) 78 { 79 struct dmaengine_buffer *dmaengine_buffer = 80 iio_buffer_to_dmaengine_buffer(&queue->buffer); 81 struct dma_async_tx_descriptor *desc; 82 enum dma_transfer_direction dma_dir; 83 struct scatterlist *sgl; 84 struct dma_vec *vecs; 85 size_t max_size; 86 dma_cookie_t cookie; 87 size_t len_total; 88 unsigned int i; 89 int nents; 90 91 max_size = min(block->size, dmaengine_buffer->max_size); 92 max_size = round_down(max_size, dmaengine_buffer->align); 93 94 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) 95 dma_dir = DMA_DEV_TO_MEM; 96 else 97 dma_dir = DMA_MEM_TO_DEV; 98 99 if (block->cyclic && iio_dmaengine_buffer_has_active_cyclic(dmaengine_buffer)) { 100 dev_err(queue->dev, "cyclic DMA transfer already active\n"); 101 return -EBUSY; 102 } 103 104 if (block->sg_table) { 105 unsigned long flags; 106 107 sgl = block->sg_table->sgl; 108 nents = sg_nents_for_len(sgl, block->bytes_used); 109 if (nents < 0) 110 return nents; 111 112 vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC); 113 if (!vecs) 114 return -ENOMEM; 115 116 len_total = block->bytes_used; 117 118 for (i = 0; i < nents; i++) { 119 vecs[i].addr = sg_dma_address(sgl); 120 vecs[i].len = min(sg_dma_len(sgl), len_total); 121 len_total -= vecs[i].len; 122 123 sgl = sg_next(sgl); 124 } 125 126 if (block->cyclic) 127 flags = DMA_PREP_REPEAT; 128 else 129 flags = DMA_PREP_INTERRUPT; 130 131 /* 132 * A new transfer may need to end an already active cyclic transfer 133 * before it can run, so always set the EOT flag. 134 */ 135 flags |= DMA_PREP_LOAD_EOT; 136 desc = dmaengine_prep_peripheral_dma_vec(dmaengine_buffer->chan, 137 vecs, nents, dma_dir, 138 flags); 139 kfree(vecs); 140 } else { 141 max_size = min(block->size, dmaengine_buffer->max_size); 142 max_size = round_down(max_size, dmaengine_buffer->align); 143 144 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) 145 block->bytes_used = max_size; 146 147 if (!block->bytes_used || block->bytes_used > max_size) 148 return -EINVAL; 149 150 desc = dmaengine_prep_slave_single(dmaengine_buffer->chan, 151 block->phys_addr, 152 block->bytes_used, 153 dma_dir, 154 DMA_PREP_INTERRUPT); 155 } 156 if (!desc) 157 return -ENOMEM; 158 159 if (!block->cyclic) { 160 desc->callback_result = iio_dmaengine_buffer_block_done; 161 desc->callback_param = block; 162 } 163 164 cookie = dmaengine_submit(desc); 165 if (dma_submit_error(cookie)) 166 return dma_submit_error(cookie); 167 168 scoped_guard(spinlock_irq, &dmaengine_buffer->queue.list_lock) 169 list_add_tail(&block->head, &dmaengine_buffer->active); 170 171 dma_async_issue_pending(dmaengine_buffer->chan); 172 173 return 0; 174 } 175 176 static void iio_dmaengine_buffer_abort(struct iio_dma_buffer_queue *queue) 177 { 178 struct dmaengine_buffer *dmaengine_buffer = 179 iio_buffer_to_dmaengine_buffer(&queue->buffer); 180 181 dmaengine_terminate_sync(dmaengine_buffer->chan); 182 iio_dma_buffer_block_list_abort(queue, &dmaengine_buffer->active); 183 } 184 185 static void iio_dmaengine_buffer_release(struct iio_buffer *buf) 186 { 187 struct dmaengine_buffer *dmaengine_buffer = 188 iio_buffer_to_dmaengine_buffer(buf); 189 190 iio_dma_buffer_release(&dmaengine_buffer->queue); 191 kfree(dmaengine_buffer); 192 } 193 194 static const struct iio_buffer_access_funcs iio_dmaengine_buffer_ops = { 195 .read = iio_dma_buffer_read, 196 .write = iio_dma_buffer_write, 197 .set_bytes_per_datum = iio_dma_buffer_set_bytes_per_datum, 198 .set_length = iio_dma_buffer_set_length, 199 .request_update = iio_dma_buffer_request_update, 200 .enable = iio_dma_buffer_enable, 201 .disable = iio_dma_buffer_disable, 202 .data_available = iio_dma_buffer_usage, 203 .space_available = iio_dma_buffer_usage, 204 .release = iio_dmaengine_buffer_release, 205 206 .enqueue_dmabuf = iio_dma_buffer_enqueue_dmabuf, 207 .attach_dmabuf = iio_dma_buffer_attach_dmabuf, 208 .detach_dmabuf = iio_dma_buffer_detach_dmabuf, 209 210 .lock_queue = iio_dma_buffer_lock_queue, 211 .unlock_queue = iio_dma_buffer_unlock_queue, 212 213 .get_dma_dev = iio_dma_buffer_get_dma_dev, 214 215 .modes = INDIO_BUFFER_HARDWARE, 216 .flags = INDIO_BUFFER_FLAG_FIXED_WATERMARK, 217 }; 218 219 static const struct iio_dma_buffer_ops iio_dmaengine_default_ops = { 220 .submit = iio_dmaengine_buffer_submit_block, 221 .abort = iio_dmaengine_buffer_abort, 222 }; 223 224 static ssize_t iio_dmaengine_buffer_get_length_align(struct device *dev, 225 struct device_attribute *attr, char *buf) 226 { 227 struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer; 228 struct dmaengine_buffer *dmaengine_buffer = 229 iio_buffer_to_dmaengine_buffer(buffer); 230 231 return sysfs_emit(buf, "%zu\n", dmaengine_buffer->align); 232 } 233 234 static IIO_DEVICE_ATTR(length_align_bytes, 0444, 235 iio_dmaengine_buffer_get_length_align, NULL, 0); 236 237 static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = { 238 &iio_dev_attr_length_align_bytes, 239 NULL, 240 }; 241 242 /** 243 * iio_dmaengine_buffer_alloc() - Allocate new buffer which uses DMAengine 244 * @chan: DMA channel. 245 * 246 * This allocates a new IIO buffer which internally uses the DMAengine framework 247 * to perform its transfers. 248 * 249 * Once done using the buffer iio_dmaengine_buffer_free() should be used to 250 * release it. 251 */ 252 static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan) 253 { 254 struct dmaengine_buffer *dmaengine_buffer; 255 unsigned int width, src_width, dest_width; 256 struct dma_slave_caps caps; 257 int ret; 258 259 ret = dma_get_slave_caps(chan, &caps); 260 if (ret < 0) 261 return ERR_PTR(ret); 262 263 dmaengine_buffer = kzalloc_obj(*dmaengine_buffer); 264 if (!dmaengine_buffer) 265 return ERR_PTR(-ENOMEM); 266 267 /* Needs to be aligned to the maximum of the minimums */ 268 if (caps.src_addr_widths) 269 src_width = __ffs(caps.src_addr_widths); 270 else 271 src_width = 1; 272 if (caps.dst_addr_widths) 273 dest_width = __ffs(caps.dst_addr_widths); 274 else 275 dest_width = 1; 276 width = max(src_width, dest_width); 277 278 INIT_LIST_HEAD(&dmaengine_buffer->active); 279 dmaengine_buffer->chan = chan; 280 dmaengine_buffer->align = width; 281 dmaengine_buffer->max_size = dma_get_max_seg_size(chan->device->dev); 282 283 iio_dma_buffer_init(&dmaengine_buffer->queue, chan->device->dev, 284 &iio_dmaengine_default_ops); 285 286 dmaengine_buffer->queue.buffer.attrs = iio_dmaengine_buffer_attrs; 287 dmaengine_buffer->queue.buffer.access = &iio_dmaengine_buffer_ops; 288 289 return &dmaengine_buffer->queue.buffer; 290 } 291 292 /** 293 * iio_dmaengine_buffer_free() - Free dmaengine buffer 294 * @buffer: Buffer to free 295 * 296 * Frees a buffer previously allocated with iio_dmaengine_buffer_alloc(). 297 */ 298 static void iio_dmaengine_buffer_free(struct iio_buffer *buffer) 299 { 300 struct dmaengine_buffer *dmaengine_buffer = 301 iio_buffer_to_dmaengine_buffer(buffer); 302 303 iio_dma_buffer_exit(&dmaengine_buffer->queue); 304 iio_buffer_put(buffer); 305 } 306 307 /** 308 * iio_dmaengine_buffer_teardown() - Releases DMA channel and frees buffer 309 * @buffer: Buffer to free 310 * 311 * Releases the DMA channel and frees the buffer previously setup with 312 * iio_dmaengine_buffer_setup_ext(). 313 */ 314 void iio_dmaengine_buffer_teardown(struct iio_buffer *buffer) 315 { 316 struct dmaengine_buffer *dmaengine_buffer = 317 iio_buffer_to_dmaengine_buffer(buffer); 318 struct dma_chan *chan = dmaengine_buffer->chan; 319 320 iio_dmaengine_buffer_free(buffer); 321 dma_release_channel(chan); 322 } 323 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_teardown, "IIO_DMAENGINE_BUFFER"); 324 325 static struct iio_buffer 326 *__iio_dmaengine_buffer_setup_ext(struct iio_dev *indio_dev, 327 struct dma_chan *chan, 328 enum iio_buffer_direction dir) 329 { 330 struct iio_buffer *buffer; 331 int ret; 332 333 buffer = iio_dmaengine_buffer_alloc(chan); 334 if (IS_ERR(buffer)) 335 return ERR_CAST(buffer); 336 337 indio_dev->modes |= INDIO_BUFFER_HARDWARE; 338 339 buffer->direction = dir; 340 341 ret = iio_device_attach_buffer(indio_dev, buffer); 342 if (ret) { 343 iio_dmaengine_buffer_free(buffer); 344 return ERR_PTR(ret); 345 } 346 347 return buffer; 348 } 349 350 /** 351 * iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device 352 * @dev: DMA channel consumer device 353 * @indio_dev: IIO device to which to attach this buffer. 354 * @channel: DMA channel name, typically "rx". 355 * @dir: Direction of buffer (in or out) 356 * 357 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 358 * and attaches it to an IIO device with iio_device_attach_buffer(). 359 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 360 * IIO device. 361 * 362 * Once done using the buffer iio_dmaengine_buffer_teardown() should be used to 363 * release it. 364 */ 365 struct iio_buffer *iio_dmaengine_buffer_setup_ext(struct device *dev, 366 struct iio_dev *indio_dev, 367 const char *channel, 368 enum iio_buffer_direction dir) 369 { 370 struct dma_chan *chan; 371 struct iio_buffer *buffer; 372 373 chan = dma_request_chan(dev, channel); 374 if (IS_ERR(chan)) 375 return ERR_CAST(chan); 376 377 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir); 378 if (IS_ERR(buffer)) 379 dma_release_channel(chan); 380 381 return buffer; 382 } 383 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER"); 384 385 static void devm_iio_dmaengine_buffer_teardown(void *buffer) 386 { 387 iio_dmaengine_buffer_teardown(buffer); 388 } 389 390 /** 391 * devm_iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device 392 * @dev: Device for devm ownership and DMA channel consumer device 393 * @indio_dev: IIO device to which to attach this buffer. 394 * @channel: DMA channel name, typically "rx". 395 * @dir: Direction of buffer (in or out) 396 * 397 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 398 * and attaches it to an IIO device with iio_device_attach_buffer(). 399 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 400 * IIO device. 401 */ 402 int devm_iio_dmaengine_buffer_setup_ext(struct device *dev, 403 struct iio_dev *indio_dev, 404 const char *channel, 405 enum iio_buffer_direction dir) 406 { 407 struct iio_buffer *buffer; 408 409 buffer = iio_dmaengine_buffer_setup_ext(dev, indio_dev, channel, dir); 410 if (IS_ERR(buffer)) 411 return PTR_ERR(buffer); 412 413 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_teardown, 414 buffer); 415 } 416 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER"); 417 418 static void devm_iio_dmaengine_buffer_free(void *buffer) 419 { 420 iio_dmaengine_buffer_free(buffer); 421 } 422 423 /** 424 * devm_iio_dmaengine_buffer_setup_with_handle() - Setup a DMA buffer for an 425 * IIO device 426 * @dev: Device for devm ownership 427 * @indio_dev: IIO device to which to attach this buffer. 428 * @chan: DMA channel 429 * @dir: Direction of buffer (in or out) 430 * 431 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 432 * and attaches it to an IIO device with iio_device_attach_buffer(). 433 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 434 * IIO device. 435 * 436 * This is the same as devm_iio_dmaengine_buffer_setup_ext() except that the 437 * caller manages requesting and releasing the DMA channel handle. 438 */ 439 int devm_iio_dmaengine_buffer_setup_with_handle(struct device *dev, 440 struct iio_dev *indio_dev, 441 struct dma_chan *chan, 442 enum iio_buffer_direction dir) 443 { 444 struct iio_buffer *buffer; 445 446 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir); 447 if (IS_ERR(buffer)) 448 return PTR_ERR(buffer); 449 450 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_free, 451 buffer); 452 } 453 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_with_handle, 454 "IIO_DMAENGINE_BUFFER"); 455 456 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 457 MODULE_DESCRIPTION("DMA buffer for the IIO framework"); 458 MODULE_LICENSE("GPL"); 459 MODULE_IMPORT_NS("IIO_DMA_BUFFER"); 460