1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2014-2015 Analog Devices Inc. 4 * Author: Lars-Peter Clausen <lars@metafoo.de> 5 */ 6 7 #include <linux/slab.h> 8 #include <linux/kernel.h> 9 #include <linux/cleanup.h> 10 #include <linux/dmaengine.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/spinlock.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 16 #include <linux/iio/iio.h> 17 #include <linux/iio/sysfs.h> 18 #include <linux/iio/buffer.h> 19 #include <linux/iio/buffer_impl.h> 20 #include <linux/iio/buffer-dma.h> 21 #include <linux/iio/buffer-dmaengine.h> 22 23 /* 24 * The IIO DMAengine buffer combines the generic IIO DMA buffer infrastructure 25 * with the DMAengine framework. The generic IIO DMA buffer infrastructure is 26 * used to manage the buffer memory and implement the IIO buffer operations 27 * while the DMAengine framework is used to perform the DMA transfers. Combined 28 * this results in a device independent fully functional DMA buffer 29 * implementation that can be used by device drivers for peripherals which are 30 * connected to a DMA controller which has a DMAengine driver implementation. 31 */ 32 33 struct dmaengine_buffer { 34 struct iio_dma_buffer_queue queue; 35 36 struct dma_chan *chan; 37 struct list_head active; 38 39 size_t align; 40 size_t max_size; 41 }; 42 43 static struct dmaengine_buffer *iio_buffer_to_dmaengine_buffer(struct iio_buffer *buffer) 44 { 45 return container_of(buffer, struct dmaengine_buffer, queue.buffer); 46 } 47 48 static void iio_dmaengine_buffer_block_done(void *data, 49 const struct dmaengine_result *result) 50 { 51 struct iio_dma_buffer_block *block = data; 52 53 scoped_guard(spinlock_irqsave, &block->queue->list_lock) 54 list_del(&block->head); 55 block->bytes_used -= result->residue; 56 iio_dma_buffer_block_done(block); 57 } 58 59 /* 60 * Cyclic transfers run until abort. Cap active cyclic blocks to one until there 61 * is a use case for more. 62 */ 63 static bool iio_dmaengine_buffer_has_active_cyclic(struct dmaengine_buffer *dmaengine_buffer) 64 { 65 struct iio_dma_buffer_block *block; 66 67 guard(spinlock_irqsave)(&dmaengine_buffer->queue.list_lock); 68 list_for_each_entry(block, &dmaengine_buffer->active, head) { 69 if (block->cyclic) 70 return true; 71 } 72 73 return false; 74 } 75 76 static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue, 77 struct iio_dma_buffer_block *block) 78 { 79 struct dmaengine_buffer *dmaengine_buffer = 80 iio_buffer_to_dmaengine_buffer(&queue->buffer); 81 struct dma_async_tx_descriptor *desc; 82 enum dma_transfer_direction dma_dir; 83 struct scatterlist *sgl; 84 struct dma_vec *vecs; 85 size_t max_size; 86 dma_cookie_t cookie; 87 size_t len_total; 88 unsigned int i; 89 int nents; 90 91 max_size = min(block->size, dmaengine_buffer->max_size); 92 max_size = round_down(max_size, dmaengine_buffer->align); 93 94 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) 95 dma_dir = DMA_DEV_TO_MEM; 96 else 97 dma_dir = DMA_MEM_TO_DEV; 98 99 if (block->cyclic && iio_dmaengine_buffer_has_active_cyclic(dmaengine_buffer)) { 100 dev_err(queue->dev, "cyclic DMA transfer already active\n"); 101 return -EBUSY; 102 } 103 104 if (block->sg_table) { 105 unsigned long flags; 106 107 /* 108 * Only the first sgt->nents entries carry a valid 109 * sg_dma_address()/sg_dma_len() pair as mapping the table may 110 * have coalesced entries, in which case nents is smaller than 111 * orig_nents. 112 */ 113 nents = block->sg_table->nents; 114 115 vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC); 116 if (!vecs) 117 return -ENOMEM; 118 119 len_total = block->bytes_used; 120 121 sgl = block->sg_table->sgl; 122 for (i = 0; i < nents && len_total; i++) { 123 vecs[i].addr = sg_dma_address(sgl); 124 vecs[i].len = min(sg_dma_len(sgl), len_total); 125 len_total -= vecs[i].len; 126 127 sgl = sg_next(sgl); 128 } 129 130 nents = i; 131 132 if (block->cyclic) 133 flags = DMA_PREP_REPEAT; 134 else 135 flags = DMA_PREP_INTERRUPT; 136 137 /* 138 * A new transfer may need to end an already active cyclic transfer 139 * before it can run, so always set the EOT flag. 140 */ 141 flags |= DMA_PREP_LOAD_EOT; 142 desc = dmaengine_prep_peripheral_dma_vec(dmaengine_buffer->chan, 143 vecs, nents, dma_dir, 144 flags); 145 kfree(vecs); 146 } else { 147 max_size = min(block->size, dmaengine_buffer->max_size); 148 max_size = round_down(max_size, dmaengine_buffer->align); 149 150 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) 151 block->bytes_used = max_size; 152 153 if (!block->bytes_used || block->bytes_used > max_size) 154 return -EINVAL; 155 156 desc = dmaengine_prep_slave_single(dmaengine_buffer->chan, 157 block->phys_addr, 158 block->bytes_used, 159 dma_dir, 160 DMA_PREP_INTERRUPT); 161 } 162 if (!desc) 163 return -ENOMEM; 164 165 if (!block->cyclic) { 166 desc->callback_result = iio_dmaengine_buffer_block_done; 167 desc->callback_param = block; 168 } 169 170 cookie = dmaengine_submit(desc); 171 if (dma_submit_error(cookie)) 172 return dma_submit_error(cookie); 173 174 scoped_guard(spinlock_irq, &dmaengine_buffer->queue.list_lock) 175 list_add_tail(&block->head, &dmaengine_buffer->active); 176 177 dma_async_issue_pending(dmaengine_buffer->chan); 178 179 return 0; 180 } 181 182 static void iio_dmaengine_buffer_abort(struct iio_dma_buffer_queue *queue) 183 { 184 struct dmaengine_buffer *dmaengine_buffer = 185 iio_buffer_to_dmaengine_buffer(&queue->buffer); 186 187 dmaengine_terminate_sync(dmaengine_buffer->chan); 188 iio_dma_buffer_block_list_abort(queue, &dmaengine_buffer->active); 189 } 190 191 static void iio_dmaengine_buffer_release(struct iio_buffer *buf) 192 { 193 struct dmaengine_buffer *dmaengine_buffer = 194 iio_buffer_to_dmaengine_buffer(buf); 195 196 iio_dma_buffer_release(&dmaengine_buffer->queue); 197 kfree(dmaengine_buffer); 198 } 199 200 static const struct iio_buffer_access_funcs iio_dmaengine_buffer_ops = { 201 .read = iio_dma_buffer_read, 202 .write = iio_dma_buffer_write, 203 .set_bytes_per_datum = iio_dma_buffer_set_bytes_per_datum, 204 .set_length = iio_dma_buffer_set_length, 205 .request_update = iio_dma_buffer_request_update, 206 .enable = iio_dma_buffer_enable, 207 .disable = iio_dma_buffer_disable, 208 .data_available = iio_dma_buffer_usage, 209 .space_available = iio_dma_buffer_usage, 210 .release = iio_dmaengine_buffer_release, 211 212 .enqueue_dmabuf = iio_dma_buffer_enqueue_dmabuf, 213 .attach_dmabuf = iio_dma_buffer_attach_dmabuf, 214 .detach_dmabuf = iio_dma_buffer_detach_dmabuf, 215 216 .lock_queue = iio_dma_buffer_lock_queue, 217 .unlock_queue = iio_dma_buffer_unlock_queue, 218 219 .get_dma_dev = iio_dma_buffer_get_dma_dev, 220 221 .modes = INDIO_BUFFER_HARDWARE, 222 .flags = INDIO_BUFFER_FLAG_FIXED_WATERMARK, 223 }; 224 225 static const struct iio_dma_buffer_ops iio_dmaengine_default_ops = { 226 .submit = iio_dmaengine_buffer_submit_block, 227 .abort = iio_dmaengine_buffer_abort, 228 }; 229 230 static ssize_t iio_dmaengine_buffer_get_length_align(struct device *dev, 231 struct device_attribute *attr, char *buf) 232 { 233 struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer; 234 struct dmaengine_buffer *dmaengine_buffer = 235 iio_buffer_to_dmaengine_buffer(buffer); 236 237 return sysfs_emit(buf, "%zu\n", dmaengine_buffer->align); 238 } 239 240 static IIO_DEVICE_ATTR(length_align_bytes, 0444, 241 iio_dmaengine_buffer_get_length_align, NULL, 0); 242 243 static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = { 244 &iio_dev_attr_length_align_bytes, 245 NULL, 246 }; 247 248 /** 249 * iio_dmaengine_buffer_alloc() - Allocate new buffer which uses DMAengine 250 * @chan: DMA channel. 251 * 252 * This allocates a new IIO buffer which internally uses the DMAengine framework 253 * to perform its transfers. 254 * 255 * Once done using the buffer iio_dmaengine_buffer_free() should be used to 256 * release it. 257 */ 258 static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan) 259 { 260 struct dmaengine_buffer *dmaengine_buffer; 261 unsigned int width, src_width, dest_width; 262 struct dma_slave_caps caps; 263 int ret; 264 265 ret = dma_get_slave_caps(chan, &caps); 266 if (ret < 0) 267 return ERR_PTR(ret); 268 269 dmaengine_buffer = kzalloc_obj(*dmaengine_buffer); 270 if (!dmaengine_buffer) 271 return ERR_PTR(-ENOMEM); 272 273 /* Needs to be aligned to the maximum of the minimums */ 274 if (caps.src_addr_widths) 275 src_width = __ffs(caps.src_addr_widths); 276 else 277 src_width = 1; 278 if (caps.dst_addr_widths) 279 dest_width = __ffs(caps.dst_addr_widths); 280 else 281 dest_width = 1; 282 width = max(src_width, dest_width); 283 284 INIT_LIST_HEAD(&dmaengine_buffer->active); 285 dmaengine_buffer->chan = chan; 286 dmaengine_buffer->align = width; 287 dmaengine_buffer->max_size = dma_get_max_seg_size(chan->device->dev); 288 289 iio_dma_buffer_init(&dmaengine_buffer->queue, chan->device->dev, 290 &iio_dmaengine_default_ops); 291 292 dmaengine_buffer->queue.buffer.attrs = iio_dmaengine_buffer_attrs; 293 dmaengine_buffer->queue.buffer.access = &iio_dmaengine_buffer_ops; 294 295 return &dmaengine_buffer->queue.buffer; 296 } 297 298 /** 299 * iio_dmaengine_buffer_free() - Free dmaengine buffer 300 * @buffer: Buffer to free 301 * 302 * Frees a buffer previously allocated with iio_dmaengine_buffer_alloc(). 303 */ 304 static void iio_dmaengine_buffer_free(struct iio_buffer *buffer) 305 { 306 struct dmaengine_buffer *dmaengine_buffer = 307 iio_buffer_to_dmaengine_buffer(buffer); 308 309 iio_dma_buffer_exit(&dmaengine_buffer->queue); 310 iio_buffer_put(buffer); 311 } 312 313 /** 314 * iio_dmaengine_buffer_teardown() - Releases DMA channel and frees buffer 315 * @buffer: Buffer to free 316 * 317 * Releases the DMA channel and frees the buffer previously setup with 318 * iio_dmaengine_buffer_setup_ext(). 319 */ 320 void iio_dmaengine_buffer_teardown(struct iio_buffer *buffer) 321 { 322 struct dmaengine_buffer *dmaengine_buffer = 323 iio_buffer_to_dmaengine_buffer(buffer); 324 struct dma_chan *chan = dmaengine_buffer->chan; 325 326 iio_dmaengine_buffer_free(buffer); 327 dma_release_channel(chan); 328 } 329 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_teardown, "IIO_DMAENGINE_BUFFER"); 330 331 static struct iio_buffer 332 *__iio_dmaengine_buffer_setup_ext(struct iio_dev *indio_dev, 333 struct dma_chan *chan, 334 enum iio_buffer_direction dir) 335 { 336 struct iio_buffer *buffer; 337 int ret; 338 339 buffer = iio_dmaengine_buffer_alloc(chan); 340 if (IS_ERR(buffer)) 341 return ERR_CAST(buffer); 342 343 indio_dev->modes |= INDIO_BUFFER_HARDWARE; 344 345 buffer->direction = dir; 346 347 ret = iio_device_attach_buffer(indio_dev, buffer); 348 if (ret) { 349 iio_dmaengine_buffer_free(buffer); 350 return ERR_PTR(ret); 351 } 352 353 return buffer; 354 } 355 356 /** 357 * iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device 358 * @dev: DMA channel consumer device 359 * @indio_dev: IIO device to which to attach this buffer. 360 * @channel: DMA channel name, typically "rx". 361 * @dir: Direction of buffer (in or out) 362 * 363 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 364 * and attaches it to an IIO device with iio_device_attach_buffer(). 365 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 366 * IIO device. 367 * 368 * Once done using the buffer iio_dmaengine_buffer_teardown() should be used to 369 * release it. 370 */ 371 struct iio_buffer *iio_dmaengine_buffer_setup_ext(struct device *dev, 372 struct iio_dev *indio_dev, 373 const char *channel, 374 enum iio_buffer_direction dir) 375 { 376 struct dma_chan *chan; 377 struct iio_buffer *buffer; 378 379 chan = dma_request_chan(dev, channel); 380 if (IS_ERR(chan)) 381 return ERR_CAST(chan); 382 383 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir); 384 if (IS_ERR(buffer)) 385 dma_release_channel(chan); 386 387 return buffer; 388 } 389 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER"); 390 391 static void devm_iio_dmaengine_buffer_teardown(void *buffer) 392 { 393 iio_dmaengine_buffer_teardown(buffer); 394 } 395 396 /** 397 * devm_iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device 398 * @dev: Device for devm ownership and DMA channel consumer device 399 * @indio_dev: IIO device to which to attach this buffer. 400 * @channel: DMA channel name, typically "rx". 401 * @dir: Direction of buffer (in or out) 402 * 403 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 404 * and attaches it to an IIO device with iio_device_attach_buffer(). 405 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 406 * IIO device. 407 */ 408 int devm_iio_dmaengine_buffer_setup_ext(struct device *dev, 409 struct iio_dev *indio_dev, 410 const char *channel, 411 enum iio_buffer_direction dir) 412 { 413 struct iio_buffer *buffer; 414 415 buffer = iio_dmaengine_buffer_setup_ext(dev, indio_dev, channel, dir); 416 if (IS_ERR(buffer)) 417 return PTR_ERR(buffer); 418 419 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_teardown, 420 buffer); 421 } 422 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER"); 423 424 static void devm_iio_dmaengine_buffer_free(void *buffer) 425 { 426 iio_dmaengine_buffer_free(buffer); 427 } 428 429 /** 430 * devm_iio_dmaengine_buffer_setup_with_handle() - Setup a DMA buffer for an 431 * IIO device 432 * @dev: Device for devm ownership 433 * @indio_dev: IIO device to which to attach this buffer. 434 * @chan: DMA channel 435 * @dir: Direction of buffer (in or out) 436 * 437 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 438 * and attaches it to an IIO device with iio_device_attach_buffer(). 439 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 440 * IIO device. 441 * 442 * This is the same as devm_iio_dmaengine_buffer_setup_ext() except that the 443 * caller manages requesting and releasing the DMA channel handle. 444 */ 445 int devm_iio_dmaengine_buffer_setup_with_handle(struct device *dev, 446 struct iio_dev *indio_dev, 447 struct dma_chan *chan, 448 enum iio_buffer_direction dir) 449 { 450 struct iio_buffer *buffer; 451 452 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir); 453 if (IS_ERR(buffer)) 454 return PTR_ERR(buffer); 455 456 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_free, 457 buffer); 458 } 459 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_with_handle, 460 "IIO_DMAENGINE_BUFFER"); 461 462 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 463 MODULE_DESCRIPTION("DMA buffer for the IIO framework"); 464 MODULE_LICENSE("GPL"); 465 MODULE_IMPORT_NS("IIO_DMA_BUFFER"); 466