1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2014-2015 Analog Devices Inc. 4 * Author: Lars-Peter Clausen <lars@metafoo.de> 5 */ 6 7 #include <linux/slab.h> 8 #include <linux/kernel.h> 9 #include <linux/cleanup.h> 10 #include <linux/dmaengine.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/spinlock.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 16 #include <linux/iio/iio.h> 17 #include <linux/iio/sysfs.h> 18 #include <linux/iio/buffer.h> 19 #include <linux/iio/buffer_impl.h> 20 #include <linux/iio/buffer-dma.h> 21 #include <linux/iio/buffer-dmaengine.h> 22 23 /* 24 * The IIO DMAengine buffer combines the generic IIO DMA buffer infrastructure 25 * with the DMAengine framework. The generic IIO DMA buffer infrastructure is 26 * used to manage the buffer memory and implement the IIO buffer operations 27 * while the DMAengine framework is used to perform the DMA transfers. Combined 28 * this results in a device independent fully functional DMA buffer 29 * implementation that can be used by device drivers for peripherals which are 30 * connected to a DMA controller which has a DMAengine driver implementation. 31 */ 32 33 struct dmaengine_buffer { 34 struct iio_dma_buffer_queue queue; 35 36 struct dma_chan *chan; 37 struct list_head active; 38 39 size_t align; 40 size_t max_size; 41 }; 42 43 static struct dmaengine_buffer *iio_buffer_to_dmaengine_buffer(struct iio_buffer *buffer) 44 { 45 return container_of(buffer, struct dmaengine_buffer, queue.buffer); 46 } 47 48 static void iio_dmaengine_buffer_block_done(void *data, 49 const struct dmaengine_result *result) 50 { 51 struct iio_dma_buffer_block *block = data; 52 53 scoped_guard(spinlock_irqsave, &block->queue->list_lock) 54 list_del(&block->head); 55 block->bytes_used -= result->residue; 56 iio_dma_buffer_block_done(block); 57 } 58 59 /* 60 * Cyclic transfers run until abort. Cap active cyclic blocks to one until there 61 * is a use case for more. 62 */ 63 static bool iio_dmaengine_buffer_has_active_cyclic(struct dmaengine_buffer *dmaengine_buffer) 64 { 65 struct iio_dma_buffer_block *block; 66 67 guard(spinlock_irqsave)(&dmaengine_buffer->queue.list_lock); 68 list_for_each_entry(block, &dmaengine_buffer->active, head) { 69 if (block->cyclic) 70 return true; 71 } 72 73 return false; 74 } 75 76 static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue, 77 struct iio_dma_buffer_block *block) 78 { 79 struct dmaengine_buffer *dmaengine_buffer = 80 iio_buffer_to_dmaengine_buffer(&queue->buffer); 81 struct dma_async_tx_descriptor *desc; 82 enum dma_transfer_direction dma_dir; 83 struct scatterlist *sgl; 84 struct dma_vec *vecs; 85 size_t max_size; 86 dma_cookie_t cookie; 87 size_t len_total; 88 unsigned int i; 89 int nents; 90 91 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) 92 dma_dir = DMA_DEV_TO_MEM; 93 else 94 dma_dir = DMA_MEM_TO_DEV; 95 96 if (block->cyclic && iio_dmaengine_buffer_has_active_cyclic(dmaengine_buffer)) { 97 dev_err(queue->dev, "cyclic DMA transfer already active\n"); 98 return -EBUSY; 99 } 100 101 if (block->sg_table) { 102 unsigned long flags; 103 104 /* 105 * Only the first sgt->nents entries carry a valid 106 * sg_dma_address()/sg_dma_len() pair as mapping the table may 107 * have coalesced entries, in which case nents is smaller than 108 * orig_nents. 109 */ 110 nents = block->sg_table->nents; 111 112 vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC); 113 if (!vecs) 114 return -ENOMEM; 115 116 len_total = block->bytes_used; 117 118 sgl = block->sg_table->sgl; 119 for (i = 0; i < nents && len_total; i++) { 120 vecs[i].addr = sg_dma_address(sgl); 121 vecs[i].len = min(sg_dma_len(sgl), len_total); 122 len_total -= vecs[i].len; 123 124 sgl = sg_next(sgl); 125 } 126 127 nents = i; 128 129 if (block->cyclic) 130 flags = DMA_PREP_REPEAT; 131 else 132 flags = DMA_PREP_INTERRUPT; 133 134 /* 135 * A new transfer may need to end an already active cyclic transfer 136 * before it can run, so always set the EOT flag. 137 */ 138 flags |= DMA_PREP_LOAD_EOT; 139 desc = dmaengine_prep_peripheral_dma_vec(dmaengine_buffer->chan, 140 vecs, nents, dma_dir, 141 flags); 142 kfree(vecs); 143 } else { 144 max_size = min(block->size, dmaengine_buffer->max_size); 145 max_size = round_down(max_size, dmaengine_buffer->align); 146 147 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN) 148 block->bytes_used = max_size; 149 150 if (!block->bytes_used || block->bytes_used > max_size) 151 return -EINVAL; 152 153 desc = dmaengine_prep_slave_single(dmaengine_buffer->chan, 154 block->phys_addr, 155 block->bytes_used, 156 dma_dir, 157 DMA_PREP_INTERRUPT); 158 } 159 if (!desc) 160 return -ENOMEM; 161 162 if (!block->cyclic) { 163 desc->callback_result = iio_dmaengine_buffer_block_done; 164 desc->callback_param = block; 165 } 166 167 cookie = dmaengine_submit(desc); 168 if (dma_submit_error(cookie)) 169 return dma_submit_error(cookie); 170 171 scoped_guard(spinlock_irq, &dmaengine_buffer->queue.list_lock) 172 list_add_tail(&block->head, &dmaengine_buffer->active); 173 174 dma_async_issue_pending(dmaengine_buffer->chan); 175 176 return 0; 177 } 178 179 static void iio_dmaengine_buffer_abort(struct iio_dma_buffer_queue *queue) 180 { 181 struct dmaengine_buffer *dmaengine_buffer = 182 iio_buffer_to_dmaengine_buffer(&queue->buffer); 183 184 dmaengine_terminate_sync(dmaengine_buffer->chan); 185 iio_dma_buffer_block_list_abort(queue, &dmaengine_buffer->active); 186 } 187 188 static void iio_dmaengine_buffer_release(struct iio_buffer *buf) 189 { 190 struct dmaengine_buffer *dmaengine_buffer = 191 iio_buffer_to_dmaengine_buffer(buf); 192 193 iio_dma_buffer_release(&dmaengine_buffer->queue); 194 kfree(dmaengine_buffer); 195 } 196 197 static const struct iio_buffer_access_funcs iio_dmaengine_buffer_ops = { 198 .read = iio_dma_buffer_read, 199 .write = iio_dma_buffer_write, 200 .set_bytes_per_datum = iio_dma_buffer_set_bytes_per_datum, 201 .set_length = iio_dma_buffer_set_length, 202 .request_update = iio_dma_buffer_request_update, 203 .enable = iio_dma_buffer_enable, 204 .disable = iio_dma_buffer_disable, 205 .data_available = iio_dma_buffer_usage, 206 .space_available = iio_dma_buffer_usage, 207 .release = iio_dmaengine_buffer_release, 208 209 .enqueue_dmabuf = iio_dma_buffer_enqueue_dmabuf, 210 .attach_dmabuf = iio_dma_buffer_attach_dmabuf, 211 .detach_dmabuf = iio_dma_buffer_detach_dmabuf, 212 213 .lock_queue = iio_dma_buffer_lock_queue, 214 .unlock_queue = iio_dma_buffer_unlock_queue, 215 216 .get_dma_dev = iio_dma_buffer_get_dma_dev, 217 218 .modes = INDIO_BUFFER_HARDWARE, 219 .flags = INDIO_BUFFER_FLAG_FIXED_WATERMARK, 220 }; 221 222 static const struct iio_dma_buffer_ops iio_dmaengine_default_ops = { 223 .submit = iio_dmaengine_buffer_submit_block, 224 .abort = iio_dmaengine_buffer_abort, 225 }; 226 227 static ssize_t iio_dmaengine_buffer_get_length_align(struct device *dev, 228 struct device_attribute *attr, char *buf) 229 { 230 struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer; 231 struct dmaengine_buffer *dmaengine_buffer = 232 iio_buffer_to_dmaengine_buffer(buffer); 233 234 return sysfs_emit(buf, "%zu\n", dmaengine_buffer->align); 235 } 236 237 static IIO_DEVICE_ATTR(length_align_bytes, 0444, 238 iio_dmaengine_buffer_get_length_align, NULL, 0); 239 240 static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = { 241 &iio_dev_attr_length_align_bytes, 242 NULL, 243 }; 244 245 /** 246 * iio_dmaengine_buffer_alloc() - Allocate new buffer which uses DMAengine 247 * @chan: DMA channel. 248 * 249 * This allocates a new IIO buffer which internally uses the DMAengine framework 250 * to perform its transfers. 251 * 252 * Once done using the buffer iio_dmaengine_buffer_free() should be used to 253 * release it. 254 */ 255 static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan) 256 { 257 struct device *dma_dev = dmaengine_get_dma_device(chan); 258 struct dmaengine_buffer *dmaengine_buffer; 259 unsigned int width, src_width, dest_width; 260 struct dma_slave_caps caps; 261 int ret; 262 263 ret = dma_get_slave_caps(chan, &caps); 264 if (ret < 0) 265 return ERR_PTR(ret); 266 267 dmaengine_buffer = kzalloc_obj(*dmaengine_buffer); 268 if (!dmaengine_buffer) 269 return ERR_PTR(-ENOMEM); 270 271 /* Needs to be aligned to the maximum of the minimums */ 272 if (caps.src_addr_widths) 273 src_width = __ffs(caps.src_addr_widths); 274 else 275 src_width = 1; 276 if (caps.dst_addr_widths) 277 dest_width = __ffs(caps.dst_addr_widths); 278 else 279 dest_width = 1; 280 width = max(src_width, dest_width); 281 282 INIT_LIST_HEAD(&dmaengine_buffer->active); 283 dmaengine_buffer->chan = chan; 284 dmaengine_buffer->align = width; 285 dmaengine_buffer->max_size = dma_get_max_seg_size(dma_dev); 286 287 iio_dma_buffer_init(&dmaengine_buffer->queue, dma_dev, 288 &iio_dmaengine_default_ops); 289 290 dmaengine_buffer->queue.buffer.attrs = iio_dmaengine_buffer_attrs; 291 dmaengine_buffer->queue.buffer.access = &iio_dmaengine_buffer_ops; 292 293 return &dmaengine_buffer->queue.buffer; 294 } 295 296 /** 297 * iio_dmaengine_buffer_free() - Free dmaengine buffer 298 * @buffer: Buffer to free 299 * 300 * Frees a buffer previously allocated with iio_dmaengine_buffer_alloc(). 301 */ 302 static void iio_dmaengine_buffer_free(struct iio_buffer *buffer) 303 { 304 struct dmaengine_buffer *dmaengine_buffer = 305 iio_buffer_to_dmaengine_buffer(buffer); 306 307 iio_dma_buffer_exit(&dmaengine_buffer->queue); 308 iio_buffer_put(buffer); 309 } 310 311 /** 312 * iio_dmaengine_buffer_teardown() - Releases DMA channel and frees buffer 313 * @buffer: Buffer to free 314 * 315 * Releases the DMA channel and frees the buffer previously setup with 316 * iio_dmaengine_buffer_setup_ext(). 317 */ 318 void iio_dmaengine_buffer_teardown(struct iio_buffer *buffer) 319 { 320 struct dmaengine_buffer *dmaengine_buffer = 321 iio_buffer_to_dmaengine_buffer(buffer); 322 struct dma_chan *chan = dmaengine_buffer->chan; 323 324 iio_dmaengine_buffer_free(buffer); 325 dma_release_channel(chan); 326 } 327 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_teardown, "IIO_DMAENGINE_BUFFER"); 328 329 static struct iio_buffer 330 *__iio_dmaengine_buffer_setup_ext(struct iio_dev *indio_dev, 331 struct dma_chan *chan, 332 enum iio_buffer_direction dir) 333 { 334 struct iio_buffer *buffer; 335 int ret; 336 337 buffer = iio_dmaengine_buffer_alloc(chan); 338 if (IS_ERR(buffer)) 339 return ERR_CAST(buffer); 340 341 indio_dev->modes |= INDIO_BUFFER_HARDWARE; 342 343 buffer->direction = dir; 344 345 ret = iio_device_attach_buffer(indio_dev, buffer); 346 if (ret) { 347 iio_dmaengine_buffer_free(buffer); 348 return ERR_PTR(ret); 349 } 350 351 return buffer; 352 } 353 354 /** 355 * iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device 356 * @dev: DMA channel consumer device 357 * @indio_dev: IIO device to which to attach this buffer. 358 * @channel: DMA channel name, typically "rx". 359 * @dir: Direction of buffer (in or out) 360 * 361 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 362 * and attaches it to an IIO device with iio_device_attach_buffer(). 363 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 364 * IIO device. 365 * 366 * Once done using the buffer iio_dmaengine_buffer_teardown() should be used to 367 * release it. 368 */ 369 struct iio_buffer *iio_dmaengine_buffer_setup_ext(struct device *dev, 370 struct iio_dev *indio_dev, 371 const char *channel, 372 enum iio_buffer_direction dir) 373 { 374 struct dma_chan *chan; 375 struct iio_buffer *buffer; 376 377 chan = dma_request_chan(dev, channel); 378 if (IS_ERR(chan)) 379 return ERR_CAST(chan); 380 381 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir); 382 if (IS_ERR(buffer)) 383 dma_release_channel(chan); 384 385 return buffer; 386 } 387 EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER"); 388 389 static void devm_iio_dmaengine_buffer_teardown(void *buffer) 390 { 391 iio_dmaengine_buffer_teardown(buffer); 392 } 393 394 /** 395 * devm_iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device 396 * @dev: Device for devm ownership and DMA channel consumer device 397 * @indio_dev: IIO device to which to attach this buffer. 398 * @channel: DMA channel name, typically "rx". 399 * @dir: Direction of buffer (in or out) 400 * 401 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 402 * and attaches it to an IIO device with iio_device_attach_buffer(). 403 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 404 * IIO device. 405 */ 406 int devm_iio_dmaengine_buffer_setup_ext(struct device *dev, 407 struct iio_dev *indio_dev, 408 const char *channel, 409 enum iio_buffer_direction dir) 410 { 411 struct iio_buffer *buffer; 412 413 buffer = iio_dmaengine_buffer_setup_ext(dev, indio_dev, channel, dir); 414 if (IS_ERR(buffer)) 415 return PTR_ERR(buffer); 416 417 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_teardown, 418 buffer); 419 } 420 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER"); 421 422 static void devm_iio_dmaengine_buffer_free(void *buffer) 423 { 424 iio_dmaengine_buffer_free(buffer); 425 } 426 427 /** 428 * devm_iio_dmaengine_buffer_setup_with_handle() - Setup a DMA buffer for an 429 * IIO device 430 * @dev: Device for devm ownership 431 * @indio_dev: IIO device to which to attach this buffer. 432 * @chan: DMA channel 433 * @dir: Direction of buffer (in or out) 434 * 435 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc() 436 * and attaches it to an IIO device with iio_device_attach_buffer(). 437 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the 438 * IIO device. 439 * 440 * This is the same as devm_iio_dmaengine_buffer_setup_ext() except that the 441 * caller manages requesting and releasing the DMA channel handle. 442 */ 443 int devm_iio_dmaengine_buffer_setup_with_handle(struct device *dev, 444 struct iio_dev *indio_dev, 445 struct dma_chan *chan, 446 enum iio_buffer_direction dir) 447 { 448 struct iio_buffer *buffer; 449 450 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir); 451 if (IS_ERR(buffer)) 452 return PTR_ERR(buffer); 453 454 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_free, 455 buffer); 456 } 457 EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_with_handle, 458 "IIO_DMAENGINE_BUFFER"); 459 460 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 461 MODULE_DESCRIPTION("DMA buffer for the IIO framework"); 462 MODULE_LICENSE("GPL"); 463 MODULE_IMPORT_NS("IIO_DMA_BUFFER"); 464