1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * BCM2835 DMA engine support 4 * 5 * This driver only supports cyclic DMA transfers 6 * as needed for the I2S module. 7 * 8 * Author: Florian Meier <florian.meier@koalo.de> 9 * Copyright 2013 10 * 11 * Based on 12 * OMAP DMAengine support by Russell King 13 * 14 * BCM2708 DMA Driver 15 * Copyright (C) 2010 Broadcom 16 * 17 * Raspberry Pi PCM I2S ALSA Driver 18 * Copyright (c) by Phil Poole 2013 19 * 20 * MARVELL MMP Peripheral DMA Driver 21 * Copyright 2012 Marvell International Ltd. 22 */ 23 #include <linux/dmaengine.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/dmapool.h> 26 #include <linux/err.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/list.h> 30 #include <linux/module.h> 31 #include <linux/platform_device.h> 32 #include <linux/slab.h> 33 #include <linux/io.h> 34 #include <linux/spinlock.h> 35 #include <linux/of.h> 36 #include <linux/of_dma.h> 37 38 #include "virt-dma.h" 39 40 #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14 41 #define BCM2835_DMA_CHAN_NAME_SIZE 8 42 43 struct bcm2835_dmadev { 44 struct dma_device ddev; 45 spinlock_t lock; 46 void __iomem *base; 47 struct device_dma_parameters dma_parms; 48 }; 49 50 struct bcm2835_dma_cb { 51 uint32_t info; 52 uint32_t src; 53 uint32_t dst; 54 uint32_t length; 55 uint32_t stride; 56 uint32_t next; 57 uint32_t pad[2]; 58 }; 59 60 struct bcm2835_cb_entry { 61 struct bcm2835_dma_cb *cb; 62 dma_addr_t paddr; 63 }; 64 65 struct bcm2835_chan { 66 struct virt_dma_chan vc; 67 struct list_head node; 68 69 struct dma_slave_config cfg; 70 unsigned int dreq; 71 72 int ch; 73 struct bcm2835_desc *desc; 74 struct dma_pool *cb_pool; 75 76 void __iomem *chan_base; 77 int irq_number; 78 unsigned int irq_flags; 79 80 bool is_lite_channel; 81 }; 82 83 struct bcm2835_desc { 84 struct bcm2835_chan *c; 85 struct virt_dma_desc vd; 86 enum dma_transfer_direction dir; 87 88 unsigned int frames; 89 size_t size; 90 91 bool cyclic; 92 93 struct bcm2835_cb_entry cb_list[]; 94 }; 95 96 #define BCM2835_DMA_CS 0x00 97 #define BCM2835_DMA_ADDR 0x04 98 #define BCM2835_DMA_TI 0x08 99 #define BCM2835_DMA_SOURCE_AD 0x0c 100 #define BCM2835_DMA_DEST_AD 0x10 101 #define BCM2835_DMA_LEN 0x14 102 #define BCM2835_DMA_STRIDE 0x18 103 #define BCM2835_DMA_NEXTCB 0x1c 104 #define BCM2835_DMA_DEBUG 0x20 105 106 /* DMA CS Control and Status bits */ 107 #define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */ 108 #define BCM2835_DMA_END BIT(1) /* current CB has ended */ 109 #define BCM2835_DMA_INT BIT(2) /* interrupt status */ 110 #define BCM2835_DMA_DREQ BIT(3) /* DREQ state */ 111 #define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */ 112 #define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */ 113 #define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last 114 * AXI-write to ack 115 */ 116 #define BCM2835_DMA_ERR BIT(8) 117 #define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */ 118 #define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */ 119 /* current value of TI.BCM2835_DMA_WAIT_RESP */ 120 #define BCM2835_DMA_WAIT_FOR_WRITES BIT(28) 121 #define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */ 122 #define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */ 123 #define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */ 124 125 /* Transfer information bits - also bcm2835_cb.info field */ 126 #define BCM2835_DMA_INT_EN BIT(0) 127 #define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */ 128 #define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */ 129 #define BCM2835_DMA_D_INC BIT(4) 130 #define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */ 131 #define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */ 132 #define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */ 133 #define BCM2835_DMA_S_INC BIT(8) 134 #define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */ 135 #define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */ 136 #define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */ 137 #define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12) 138 #define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */ 139 #define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */ 140 #define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */ 141 142 /* debug register bits */ 143 #define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0) 144 #define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1) 145 #define BCM2835_DMA_DEBUG_READ_ERR BIT(2) 146 #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4 147 #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4 148 #define BCM2835_DMA_DEBUG_ID_SHIFT 16 149 #define BCM2835_DMA_DEBUG_ID_BITS 9 150 #define BCM2835_DMA_DEBUG_STATE_SHIFT 16 151 #define BCM2835_DMA_DEBUG_STATE_BITS 9 152 #define BCM2835_DMA_DEBUG_VERSION_SHIFT 25 153 #define BCM2835_DMA_DEBUG_VERSION_BITS 3 154 #define BCM2835_DMA_DEBUG_LITE BIT(28) 155 156 /* shared registers for all dma channels */ 157 #define BCM2835_DMA_INT_STATUS 0xfe0 158 #define BCM2835_DMA_ENABLE 0xff0 159 160 #define BCM2835_DMA_DATA_TYPE_S8 1 161 #define BCM2835_DMA_DATA_TYPE_S16 2 162 #define BCM2835_DMA_DATA_TYPE_S32 4 163 #define BCM2835_DMA_DATA_TYPE_S128 16 164 165 /* Valid only for channels 0 - 14, 15 has its own base address */ 166 #define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */ 167 #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n)) 168 169 /* the max dma length for different channels */ 170 #define MAX_DMA_LEN SZ_1G 171 #define MAX_LITE_DMA_LEN (SZ_64K - 4) 172 173 static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c) 174 { 175 /* lite and normal channels have different max frame length */ 176 return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN; 177 } 178 179 /* how many frames of max_len size do we need to transfer len bytes */ 180 static inline size_t bcm2835_dma_frames_for_length(size_t len, 181 size_t max_len) 182 { 183 return DIV_ROUND_UP(len, max_len); 184 } 185 186 static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d) 187 { 188 return container_of(d, struct bcm2835_dmadev, ddev); 189 } 190 191 static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c) 192 { 193 return container_of(c, struct bcm2835_chan, vc.chan); 194 } 195 196 static inline struct bcm2835_desc *to_bcm2835_dma_desc( 197 struct dma_async_tx_descriptor *t) 198 { 199 return container_of(t, struct bcm2835_desc, vd.tx); 200 } 201 202 static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc) 203 { 204 size_t i; 205 206 for (i = 0; i < desc->frames; i++) 207 dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb, 208 desc->cb_list[i].paddr); 209 210 kfree(desc); 211 } 212 213 static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) 214 { 215 bcm2835_dma_free_cb_chain( 216 container_of(vd, struct bcm2835_desc, vd)); 217 } 218 219 static void bcm2835_dma_create_cb_set_length( 220 struct bcm2835_chan *chan, 221 struct bcm2835_dma_cb *control_block, 222 size_t len, 223 size_t period_len, 224 size_t *total_len, 225 u32 finalextrainfo) 226 { 227 size_t max_len = bcm2835_dma_max_frame_length(chan); 228 229 /* set the length taking lite-channel limitations into account */ 230 control_block->length = min_t(u32, len, max_len); 231 232 /* finished if we have no period_length */ 233 if (!period_len) 234 return; 235 236 /* 237 * period_len means: that we need to generate 238 * transfers that are terminating at every 239 * multiple of period_len - this is typically 240 * used to set the interrupt flag in info 241 * which is required during cyclic transfers 242 */ 243 244 /* have we filled in period_length yet? */ 245 if (*total_len + control_block->length < period_len) { 246 /* update number of bytes in this period so far */ 247 *total_len += control_block->length; 248 return; 249 } 250 251 /* calculate the length that remains to reach period_length */ 252 control_block->length = period_len - *total_len; 253 254 /* reset total_length for next period */ 255 *total_len = 0; 256 257 /* add extrainfo bits in info */ 258 control_block->info |= finalextrainfo; 259 } 260 261 static inline size_t bcm2835_dma_count_frames_for_sg( 262 struct bcm2835_chan *c, 263 struct scatterlist *sgl, 264 unsigned int sg_len) 265 { 266 size_t frames = 0; 267 struct scatterlist *sgent; 268 unsigned int i; 269 size_t plength = bcm2835_dma_max_frame_length(c); 270 271 for_each_sg(sgl, sgent, sg_len, i) 272 frames += bcm2835_dma_frames_for_length( 273 sg_dma_len(sgent), plength); 274 275 return frames; 276 } 277 278 /** 279 * bcm2835_dma_create_cb_chain - create a control block and fills data in 280 * 281 * @chan: the @dma_chan for which we run this 282 * @direction: the direction in which we transfer 283 * @cyclic: it is a cyclic transfer 284 * @info: the default info bits to apply per controlblock 285 * @frames: number of controlblocks to allocate 286 * @src: the src address to assign (if the S_INC bit is set 287 * in @info, then it gets incremented) 288 * @dst: the dst address to assign (if the D_INC bit is set 289 * in @info, then it gets incremented) 290 * @buf_len: the full buffer length (may also be 0) 291 * @period_len: the period length when to apply @finalextrainfo 292 * in addition to the last transfer 293 * this will also break some control-blocks early 294 * @finalextrainfo: additional bits in last controlblock 295 * (or when period_len is reached in case of cyclic) 296 * @gfp: the GFP flag to use for allocation 297 */ 298 static struct bcm2835_desc *bcm2835_dma_create_cb_chain( 299 struct dma_chan *chan, enum dma_transfer_direction direction, 300 bool cyclic, u32 info, u32 finalextrainfo, size_t frames, 301 dma_addr_t src, dma_addr_t dst, size_t buf_len, 302 size_t period_len, gfp_t gfp) 303 { 304 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 305 size_t len = buf_len, total_len; 306 size_t frame; 307 struct bcm2835_desc *d; 308 struct bcm2835_cb_entry *cb_entry; 309 struct bcm2835_dma_cb *control_block; 310 311 if (!frames) 312 return NULL; 313 314 /* allocate and setup the descriptor. */ 315 d = kzalloc(sizeof(*d) + frames * sizeof(struct bcm2835_cb_entry), 316 gfp); 317 if (!d) 318 return NULL; 319 320 d->c = c; 321 d->dir = direction; 322 d->cyclic = cyclic; 323 324 /* 325 * Iterate over all frames, create a control block 326 * for each frame and link them together. 327 */ 328 for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) { 329 cb_entry = &d->cb_list[frame]; 330 cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp, 331 &cb_entry->paddr); 332 if (!cb_entry->cb) 333 goto error_cb; 334 335 /* fill in the control block */ 336 control_block = cb_entry->cb; 337 control_block->info = info; 338 control_block->src = src; 339 control_block->dst = dst; 340 control_block->stride = 0; 341 control_block->next = 0; 342 /* set up length in control_block if requested */ 343 if (buf_len) { 344 /* calculate length honoring period_length */ 345 bcm2835_dma_create_cb_set_length( 346 c, control_block, 347 len, period_len, &total_len, 348 cyclic ? finalextrainfo : 0); 349 350 /* calculate new remaining length */ 351 len -= control_block->length; 352 } 353 354 /* link this the last controlblock */ 355 if (frame) 356 d->cb_list[frame - 1].cb->next = cb_entry->paddr; 357 358 /* update src and dst and length */ 359 if (src && (info & BCM2835_DMA_S_INC)) 360 src += control_block->length; 361 if (dst && (info & BCM2835_DMA_D_INC)) 362 dst += control_block->length; 363 364 /* Length of total transfer */ 365 d->size += control_block->length; 366 } 367 368 /* the last frame requires extra flags */ 369 d->cb_list[d->frames - 1].cb->info |= finalextrainfo; 370 371 /* detect a size missmatch */ 372 if (buf_len && (d->size != buf_len)) 373 goto error_cb; 374 375 return d; 376 error_cb: 377 bcm2835_dma_free_cb_chain(d); 378 379 return NULL; 380 } 381 382 static void bcm2835_dma_fill_cb_chain_with_sg( 383 struct dma_chan *chan, 384 enum dma_transfer_direction direction, 385 struct bcm2835_cb_entry *cb, 386 struct scatterlist *sgl, 387 unsigned int sg_len) 388 { 389 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 390 size_t len, max_len; 391 unsigned int i; 392 dma_addr_t addr; 393 struct scatterlist *sgent; 394 395 max_len = bcm2835_dma_max_frame_length(c); 396 for_each_sg(sgl, sgent, sg_len, i) { 397 for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent); 398 len > 0; 399 addr += cb->cb->length, len -= cb->cb->length, cb++) { 400 if (direction == DMA_DEV_TO_MEM) 401 cb->cb->dst = addr; 402 else 403 cb->cb->src = addr; 404 cb->cb->length = min(len, max_len); 405 } 406 } 407 } 408 409 static int bcm2835_dma_abort(struct bcm2835_chan *c) 410 { 411 void __iomem *chan_base = c->chan_base; 412 long int timeout = 10000; 413 414 /* 415 * A zero control block address means the channel is idle. 416 * (The ACTIVE flag in the CS register is not a reliable indicator.) 417 */ 418 if (!readl(chan_base + BCM2835_DMA_ADDR)) 419 return 0; 420 421 /* Write 0 to the active bit - Pause the DMA */ 422 writel(0, chan_base + BCM2835_DMA_CS); 423 424 /* Wait for any current AXI transfer to complete */ 425 while ((readl(chan_base + BCM2835_DMA_CS) & 426 BCM2835_DMA_WAITING_FOR_WRITES) && --timeout) 427 cpu_relax(); 428 429 /* Peripheral might be stuck and fail to signal AXI write responses */ 430 if (!timeout) 431 dev_err(c->vc.chan.device->dev, 432 "failed to complete outstanding writes\n"); 433 434 writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS); 435 return 0; 436 } 437 438 static void bcm2835_dma_start_desc(struct bcm2835_chan *c) 439 { 440 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 441 struct bcm2835_desc *d; 442 443 if (!vd) { 444 c->desc = NULL; 445 return; 446 } 447 448 list_del(&vd->node); 449 450 c->desc = d = to_bcm2835_dma_desc(&vd->tx); 451 452 writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR); 453 writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS); 454 } 455 456 static irqreturn_t bcm2835_dma_callback(int irq, void *data) 457 { 458 struct bcm2835_chan *c = data; 459 struct bcm2835_desc *d; 460 unsigned long flags; 461 462 /* check the shared interrupt */ 463 if (c->irq_flags & IRQF_SHARED) { 464 /* check if the interrupt is enabled */ 465 flags = readl(c->chan_base + BCM2835_DMA_CS); 466 /* if not set then we are not the reason for the irq */ 467 if (!(flags & BCM2835_DMA_INT)) 468 return IRQ_NONE; 469 } 470 471 spin_lock_irqsave(&c->vc.lock, flags); 472 473 /* 474 * Clear the INT flag to receive further interrupts. Keep the channel 475 * active in case the descriptor is cyclic or in case the client has 476 * already terminated the descriptor and issued a new one. (May happen 477 * if this IRQ handler is threaded.) If the channel is finished, it 478 * will remain idle despite the ACTIVE flag being set. 479 */ 480 writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE, 481 c->chan_base + BCM2835_DMA_CS); 482 483 d = c->desc; 484 485 if (d) { 486 if (d->cyclic) { 487 /* call the cyclic callback */ 488 vchan_cyclic_callback(&d->vd); 489 } else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) { 490 vchan_cookie_complete(&c->desc->vd); 491 bcm2835_dma_start_desc(c); 492 } 493 } 494 495 spin_unlock_irqrestore(&c->vc.lock, flags); 496 497 return IRQ_HANDLED; 498 } 499 500 static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) 501 { 502 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 503 struct device *dev = c->vc.chan.device->dev; 504 505 dev_dbg(dev, "Allocating DMA channel %d\n", c->ch); 506 507 c->cb_pool = dma_pool_create(dev_name(dev), dev, 508 sizeof(struct bcm2835_dma_cb), 0, 0); 509 if (!c->cb_pool) { 510 dev_err(dev, "unable to allocate descriptor pool\n"); 511 return -ENOMEM; 512 } 513 514 return request_irq(c->irq_number, bcm2835_dma_callback, 515 c->irq_flags, "DMA IRQ", c); 516 } 517 518 static void bcm2835_dma_free_chan_resources(struct dma_chan *chan) 519 { 520 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 521 522 vchan_free_chan_resources(&c->vc); 523 free_irq(c->irq_number, c); 524 dma_pool_destroy(c->cb_pool); 525 526 dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch); 527 } 528 529 static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d) 530 { 531 return d->size; 532 } 533 534 static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr) 535 { 536 unsigned int i; 537 size_t size; 538 539 for (size = i = 0; i < d->frames; i++) { 540 struct bcm2835_dma_cb *control_block = d->cb_list[i].cb; 541 size_t this_size = control_block->length; 542 dma_addr_t dma; 543 544 if (d->dir == DMA_DEV_TO_MEM) 545 dma = control_block->dst; 546 else 547 dma = control_block->src; 548 549 if (size) 550 size += this_size; 551 else if (addr >= dma && addr < dma + this_size) 552 size += dma + this_size - addr; 553 } 554 555 return size; 556 } 557 558 static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, 559 dma_cookie_t cookie, struct dma_tx_state *txstate) 560 { 561 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 562 struct virt_dma_desc *vd; 563 enum dma_status ret; 564 unsigned long flags; 565 566 ret = dma_cookie_status(chan, cookie, txstate); 567 if (ret == DMA_COMPLETE || !txstate) 568 return ret; 569 570 spin_lock_irqsave(&c->vc.lock, flags); 571 vd = vchan_find_desc(&c->vc, cookie); 572 if (vd) { 573 txstate->residue = 574 bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx)); 575 } else if (c->desc && c->desc->vd.tx.cookie == cookie) { 576 struct bcm2835_desc *d = c->desc; 577 dma_addr_t pos; 578 579 if (d->dir == DMA_MEM_TO_DEV) 580 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD); 581 else if (d->dir == DMA_DEV_TO_MEM) 582 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD); 583 else 584 pos = 0; 585 586 txstate->residue = bcm2835_dma_desc_size_pos(d, pos); 587 } else { 588 txstate->residue = 0; 589 } 590 591 spin_unlock_irqrestore(&c->vc.lock, flags); 592 593 return ret; 594 } 595 596 static void bcm2835_dma_issue_pending(struct dma_chan *chan) 597 { 598 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 599 unsigned long flags; 600 601 spin_lock_irqsave(&c->vc.lock, flags); 602 if (vchan_issue_pending(&c->vc) && !c->desc) 603 bcm2835_dma_start_desc(c); 604 605 spin_unlock_irqrestore(&c->vc.lock, flags); 606 } 607 608 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( 609 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, 610 size_t len, unsigned long flags) 611 { 612 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 613 struct bcm2835_desc *d; 614 u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC; 615 u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP; 616 size_t max_len = bcm2835_dma_max_frame_length(c); 617 size_t frames; 618 619 /* if src, dst or len is not given return with an error */ 620 if (!src || !dst || !len) 621 return NULL; 622 623 /* calculate number of frames */ 624 frames = bcm2835_dma_frames_for_length(len, max_len); 625 626 /* allocate the CB chain - this also fills in the pointers */ 627 d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false, 628 info, extra, frames, 629 src, dst, len, 0, GFP_KERNEL); 630 if (!d) 631 return NULL; 632 633 return vchan_tx_prep(&c->vc, &d->vd, flags); 634 } 635 636 static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( 637 struct dma_chan *chan, 638 struct scatterlist *sgl, unsigned int sg_len, 639 enum dma_transfer_direction direction, 640 unsigned long flags, void *context) 641 { 642 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 643 struct bcm2835_desc *d; 644 dma_addr_t src = 0, dst = 0; 645 u32 info = BCM2835_DMA_WAIT_RESP; 646 u32 extra = BCM2835_DMA_INT_EN; 647 size_t frames; 648 649 if (!is_slave_direction(direction)) { 650 dev_err(chan->device->dev, 651 "%s: bad direction?\n", __func__); 652 return NULL; 653 } 654 655 if (c->dreq != 0) 656 info |= BCM2835_DMA_PER_MAP(c->dreq); 657 658 if (direction == DMA_DEV_TO_MEM) { 659 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 660 return NULL; 661 src = c->cfg.src_addr; 662 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; 663 } else { 664 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 665 return NULL; 666 dst = c->cfg.dst_addr; 667 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; 668 } 669 670 /* count frames in sg list */ 671 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len); 672 673 /* allocate the CB chain */ 674 d = bcm2835_dma_create_cb_chain(chan, direction, false, 675 info, extra, 676 frames, src, dst, 0, 0, 677 GFP_KERNEL); 678 if (!d) 679 return NULL; 680 681 /* fill in frames with scatterlist pointers */ 682 bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list, 683 sgl, sg_len); 684 685 return vchan_tx_prep(&c->vc, &d->vd, flags); 686 } 687 688 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( 689 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 690 size_t period_len, enum dma_transfer_direction direction, 691 unsigned long flags) 692 { 693 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 694 struct bcm2835_desc *d; 695 dma_addr_t src, dst; 696 u32 info = BCM2835_DMA_WAIT_RESP; 697 u32 extra = BCM2835_DMA_INT_EN; 698 size_t max_len = bcm2835_dma_max_frame_length(c); 699 size_t frames; 700 701 /* Grab configuration */ 702 if (!is_slave_direction(direction)) { 703 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 704 return NULL; 705 } 706 707 if (!buf_len) { 708 dev_err(chan->device->dev, 709 "%s: bad buffer length (= 0)\n", __func__); 710 return NULL; 711 } 712 713 /* 714 * warn if buf_len is not a multiple of period_len - this may leed 715 * to unexpected latencies for interrupts and thus audiable clicks 716 */ 717 if (buf_len % period_len) 718 dev_warn_once(chan->device->dev, 719 "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n", 720 __func__, buf_len, period_len); 721 722 /* Setup DREQ channel */ 723 if (c->dreq != 0) 724 info |= BCM2835_DMA_PER_MAP(c->dreq); 725 726 if (direction == DMA_DEV_TO_MEM) { 727 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 728 return NULL; 729 src = c->cfg.src_addr; 730 dst = buf_addr; 731 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; 732 } else { 733 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 734 return NULL; 735 dst = c->cfg.dst_addr; 736 src = buf_addr; 737 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; 738 } 739 740 /* calculate number of frames */ 741 frames = /* number of periods */ 742 DIV_ROUND_UP(buf_len, period_len) * 743 /* number of frames per period */ 744 bcm2835_dma_frames_for_length(period_len, max_len); 745 746 /* 747 * allocate the CB chain 748 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine 749 * implementation calls prep_dma_cyclic with interrupts disabled. 750 */ 751 d = bcm2835_dma_create_cb_chain(chan, direction, true, 752 info, extra, 753 frames, src, dst, buf_len, 754 period_len, GFP_NOWAIT); 755 if (!d) 756 return NULL; 757 758 /* wrap around into a loop */ 759 d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr; 760 761 return vchan_tx_prep(&c->vc, &d->vd, flags); 762 } 763 764 static int bcm2835_dma_slave_config(struct dma_chan *chan, 765 struct dma_slave_config *cfg) 766 { 767 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 768 769 c->cfg = *cfg; 770 771 return 0; 772 } 773 774 static int bcm2835_dma_terminate_all(struct dma_chan *chan) 775 { 776 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 777 struct bcm2835_dmadev *d = to_bcm2835_dma_dev(c->vc.chan.device); 778 unsigned long flags; 779 LIST_HEAD(head); 780 781 spin_lock_irqsave(&c->vc.lock, flags); 782 783 /* Prevent this channel being scheduled */ 784 spin_lock(&d->lock); 785 list_del_init(&c->node); 786 spin_unlock(&d->lock); 787 788 /* stop DMA activity */ 789 if (c->desc) { 790 vchan_terminate_vdesc(&c->desc->vd); 791 c->desc = NULL; 792 bcm2835_dma_abort(c); 793 } 794 795 vchan_get_all_descriptors(&c->vc, &head); 796 spin_unlock_irqrestore(&c->vc.lock, flags); 797 vchan_dma_desc_free_list(&c->vc, &head); 798 799 return 0; 800 } 801 802 static void bcm2835_dma_synchronize(struct dma_chan *chan) 803 { 804 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 805 806 vchan_synchronize(&c->vc); 807 } 808 809 static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, 810 int irq, unsigned int irq_flags) 811 { 812 struct bcm2835_chan *c; 813 814 c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL); 815 if (!c) 816 return -ENOMEM; 817 818 c->vc.desc_free = bcm2835_dma_desc_free; 819 vchan_init(&c->vc, &d->ddev); 820 INIT_LIST_HEAD(&c->node); 821 822 c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); 823 c->ch = chan_id; 824 c->irq_number = irq; 825 c->irq_flags = irq_flags; 826 827 /* check in DEBUG register if this is a LITE channel */ 828 if (readl(c->chan_base + BCM2835_DMA_DEBUG) & 829 BCM2835_DMA_DEBUG_LITE) 830 c->is_lite_channel = true; 831 832 return 0; 833 } 834 835 static void bcm2835_dma_free(struct bcm2835_dmadev *od) 836 { 837 struct bcm2835_chan *c, *next; 838 839 list_for_each_entry_safe(c, next, &od->ddev.channels, 840 vc.chan.device_node) { 841 list_del(&c->vc.chan.device_node); 842 tasklet_kill(&c->vc.task); 843 } 844 } 845 846 static const struct of_device_id bcm2835_dma_of_match[] = { 847 { .compatible = "brcm,bcm2835-dma", }, 848 {}, 849 }; 850 MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match); 851 852 static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, 853 struct of_dma *ofdma) 854 { 855 struct bcm2835_dmadev *d = ofdma->of_dma_data; 856 struct dma_chan *chan; 857 858 chan = dma_get_any_slave_channel(&d->ddev); 859 if (!chan) 860 return NULL; 861 862 /* Set DREQ from param */ 863 to_bcm2835_dma_chan(chan)->dreq = spec->args[0]; 864 865 return chan; 866 } 867 868 static int bcm2835_dma_probe(struct platform_device *pdev) 869 { 870 struct bcm2835_dmadev *od; 871 struct resource *res; 872 void __iomem *base; 873 int rc; 874 int i, j; 875 int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1]; 876 int irq_flags; 877 uint32_t chans_available; 878 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE]; 879 880 if (!pdev->dev.dma_mask) 881 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; 882 883 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 884 if (rc) 885 return rc; 886 887 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); 888 if (!od) 889 return -ENOMEM; 890 891 pdev->dev.dma_parms = &od->dma_parms; 892 dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF); 893 894 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 895 base = devm_ioremap_resource(&pdev->dev, res); 896 if (IS_ERR(base)) 897 return PTR_ERR(base); 898 899 od->base = base; 900 901 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 902 dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask); 903 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); 904 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 905 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); 906 od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources; 907 od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources; 908 od->ddev.device_tx_status = bcm2835_dma_tx_status; 909 od->ddev.device_issue_pending = bcm2835_dma_issue_pending; 910 od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic; 911 od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg; 912 od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy; 913 od->ddev.device_config = bcm2835_dma_slave_config; 914 od->ddev.device_terminate_all = bcm2835_dma_terminate_all; 915 od->ddev.device_synchronize = bcm2835_dma_synchronize; 916 od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 917 od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 918 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | 919 BIT(DMA_MEM_TO_MEM); 920 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 921 od->ddev.dev = &pdev->dev; 922 INIT_LIST_HEAD(&od->ddev.channels); 923 spin_lock_init(&od->lock); 924 925 platform_set_drvdata(pdev, od); 926 927 /* Request DMA channel mask from device tree */ 928 if (of_property_read_u32(pdev->dev.of_node, 929 "brcm,dma-channel-mask", 930 &chans_available)) { 931 dev_err(&pdev->dev, "Failed to get channel mask\n"); 932 rc = -EINVAL; 933 goto err_no_dma; 934 } 935 936 /* get irqs for each channel that we support */ 937 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { 938 /* skip masked out channels */ 939 if (!(chans_available & (1 << i))) { 940 irq[i] = -1; 941 continue; 942 } 943 944 /* get the named irq */ 945 snprintf(chan_name, sizeof(chan_name), "dma%i", i); 946 irq[i] = platform_get_irq_byname(pdev, chan_name); 947 if (irq[i] >= 0) 948 continue; 949 950 /* legacy device tree case handling */ 951 dev_warn_once(&pdev->dev, 952 "missing interrupt-names property in device tree - legacy interpretation is used\n"); 953 /* 954 * in case of channel >= 11 955 * use the 11th interrupt and that is shared 956 */ 957 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11); 958 } 959 960 /* get irqs for each channel */ 961 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { 962 /* skip channels without irq */ 963 if (irq[i] < 0) 964 continue; 965 966 /* check if there are other channels that also use this irq */ 967 irq_flags = 0; 968 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++) 969 if ((i != j) && (irq[j] == irq[i])) { 970 irq_flags = IRQF_SHARED; 971 break; 972 } 973 974 /* initialize the channel */ 975 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags); 976 if (rc) 977 goto err_no_dma; 978 } 979 980 dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i); 981 982 /* Device-tree DMA controller registration */ 983 rc = of_dma_controller_register(pdev->dev.of_node, 984 bcm2835_dma_xlate, od); 985 if (rc) { 986 dev_err(&pdev->dev, "Failed to register DMA controller\n"); 987 goto err_no_dma; 988 } 989 990 rc = dma_async_device_register(&od->ddev); 991 if (rc) { 992 dev_err(&pdev->dev, 993 "Failed to register slave DMA engine device: %d\n", rc); 994 goto err_no_dma; 995 } 996 997 dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n"); 998 999 return 0; 1000 1001 err_no_dma: 1002 bcm2835_dma_free(od); 1003 return rc; 1004 } 1005 1006 static int bcm2835_dma_remove(struct platform_device *pdev) 1007 { 1008 struct bcm2835_dmadev *od = platform_get_drvdata(pdev); 1009 1010 dma_async_device_unregister(&od->ddev); 1011 bcm2835_dma_free(od); 1012 1013 return 0; 1014 } 1015 1016 static struct platform_driver bcm2835_dma_driver = { 1017 .probe = bcm2835_dma_probe, 1018 .remove = bcm2835_dma_remove, 1019 .driver = { 1020 .name = "bcm2835-dma", 1021 .of_match_table = of_match_ptr(bcm2835_dma_of_match), 1022 }, 1023 }; 1024 1025 module_platform_driver(bcm2835_dma_driver); 1026 1027 MODULE_ALIAS("platform:bcm2835-dma"); 1028 MODULE_DESCRIPTION("BCM2835 DMA engine driver"); 1029 MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); 1030 MODULE_LICENSE("GPL"); 1031