1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * BCM2835 DMA engine support 4 * 5 * Author: Florian Meier <florian.meier@koalo.de> 6 * Copyright 2013 7 * 8 * Based on 9 * OMAP DMAengine support by Russell King 10 * 11 * BCM2708 DMA Driver 12 * Copyright (C) 2010 Broadcom 13 * 14 * Raspberry Pi PCM I2S ALSA Driver 15 * Copyright (c) by Phil Poole 2013 16 * 17 * MARVELL MMP Peripheral DMA Driver 18 * Copyright 2012 Marvell International Ltd. 19 */ 20 #include <linux/dmaengine.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/dmapool.h> 23 #include <linux/err.h> 24 #include <linux/init.h> 25 #include <linux/interrupt.h> 26 #include <linux/list.h> 27 #include <linux/module.h> 28 #include <linux/platform_device.h> 29 #include <linux/slab.h> 30 #include <linux/io.h> 31 #include <linux/spinlock.h> 32 #include <linux/of.h> 33 #include <linux/of_dma.h> 34 35 #include "virt-dma.h" 36 37 #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14 38 #define BCM2835_DMA_CHAN_NAME_SIZE 8 39 40 struct bcm2835_dmadev { 41 struct dma_device ddev; 42 void __iomem *base; 43 struct device_dma_parameters dma_parms; 44 }; 45 46 struct bcm2835_dma_cb { 47 uint32_t info; 48 uint32_t src; 49 uint32_t dst; 50 uint32_t length; 51 uint32_t stride; 52 uint32_t next; 53 uint32_t pad[2]; 54 }; 55 56 struct bcm2835_cb_entry { 57 struct bcm2835_dma_cb *cb; 58 dma_addr_t paddr; 59 }; 60 61 struct bcm2835_chan { 62 struct virt_dma_chan vc; 63 64 struct dma_slave_config cfg; 65 unsigned int dreq; 66 67 int ch; 68 struct bcm2835_desc *desc; 69 struct dma_pool *cb_pool; 70 71 void __iomem *chan_base; 72 int irq_number; 73 unsigned int irq_flags; 74 75 bool is_lite_channel; 76 }; 77 78 struct bcm2835_desc { 79 struct bcm2835_chan *c; 80 struct virt_dma_desc vd; 81 enum dma_transfer_direction dir; 82 83 unsigned int frames; 84 size_t size; 85 86 bool cyclic; 87 88 struct bcm2835_cb_entry cb_list[]; 89 }; 90 91 #define BCM2835_DMA_CS 0x00 92 #define BCM2835_DMA_ADDR 0x04 93 #define BCM2835_DMA_TI 0x08 94 #define BCM2835_DMA_SOURCE_AD 0x0c 95 #define BCM2835_DMA_DEST_AD 0x10 96 #define BCM2835_DMA_LEN 0x14 97 #define BCM2835_DMA_STRIDE 0x18 98 #define BCM2835_DMA_NEXTCB 0x1c 99 #define BCM2835_DMA_DEBUG 0x20 100 101 /* DMA CS Control and Status bits */ 102 #define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */ 103 #define BCM2835_DMA_END BIT(1) /* current CB has ended */ 104 #define BCM2835_DMA_INT BIT(2) /* interrupt status */ 105 #define BCM2835_DMA_DREQ BIT(3) /* DREQ state */ 106 #define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */ 107 #define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */ 108 #define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last 109 * AXI-write to ack 110 */ 111 #define BCM2835_DMA_ERR BIT(8) 112 #define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */ 113 #define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */ 114 /* current value of TI.BCM2835_DMA_WAIT_RESP */ 115 #define BCM2835_DMA_WAIT_FOR_WRITES BIT(28) 116 #define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */ 117 #define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */ 118 #define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */ 119 120 /* Transfer information bits - also bcm2835_cb.info field */ 121 #define BCM2835_DMA_INT_EN BIT(0) 122 #define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */ 123 #define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */ 124 #define BCM2835_DMA_D_INC BIT(4) 125 #define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */ 126 #define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */ 127 #define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */ 128 #define BCM2835_DMA_S_INC BIT(8) 129 #define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */ 130 #define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */ 131 #define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */ 132 #define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12) 133 #define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */ 134 #define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */ 135 #define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */ 136 137 /* debug register bits */ 138 #define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0) 139 #define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1) 140 #define BCM2835_DMA_DEBUG_READ_ERR BIT(2) 141 #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4 142 #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4 143 #define BCM2835_DMA_DEBUG_ID_SHIFT 16 144 #define BCM2835_DMA_DEBUG_ID_BITS 9 145 #define BCM2835_DMA_DEBUG_STATE_SHIFT 16 146 #define BCM2835_DMA_DEBUG_STATE_BITS 9 147 #define BCM2835_DMA_DEBUG_VERSION_SHIFT 25 148 #define BCM2835_DMA_DEBUG_VERSION_BITS 3 149 #define BCM2835_DMA_DEBUG_LITE BIT(28) 150 151 /* shared registers for all dma channels */ 152 #define BCM2835_DMA_INT_STATUS 0xfe0 153 #define BCM2835_DMA_ENABLE 0xff0 154 155 #define BCM2835_DMA_DATA_TYPE_S8 1 156 #define BCM2835_DMA_DATA_TYPE_S16 2 157 #define BCM2835_DMA_DATA_TYPE_S32 4 158 #define BCM2835_DMA_DATA_TYPE_S128 16 159 160 /* Valid only for channels 0 - 14, 15 has its own base address */ 161 #define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */ 162 #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n)) 163 164 /* the max dma length for different channels */ 165 #define MAX_DMA_LEN SZ_1G 166 #define MAX_LITE_DMA_LEN (SZ_64K - 4) 167 168 static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c) 169 { 170 /* lite and normal channels have different max frame length */ 171 return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN; 172 } 173 174 /* how many frames of max_len size do we need to transfer len bytes */ 175 static inline size_t bcm2835_dma_frames_for_length(size_t len, 176 size_t max_len) 177 { 178 return DIV_ROUND_UP(len, max_len); 179 } 180 181 static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d) 182 { 183 return container_of(d, struct bcm2835_dmadev, ddev); 184 } 185 186 static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c) 187 { 188 return container_of(c, struct bcm2835_chan, vc.chan); 189 } 190 191 static inline struct bcm2835_desc *to_bcm2835_dma_desc( 192 struct dma_async_tx_descriptor *t) 193 { 194 return container_of(t, struct bcm2835_desc, vd.tx); 195 } 196 197 static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc) 198 { 199 size_t i; 200 201 for (i = 0; i < desc->frames; i++) 202 dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb, 203 desc->cb_list[i].paddr); 204 205 kfree(desc); 206 } 207 208 static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) 209 { 210 bcm2835_dma_free_cb_chain( 211 container_of(vd, struct bcm2835_desc, vd)); 212 } 213 214 static void bcm2835_dma_create_cb_set_length( 215 struct bcm2835_chan *chan, 216 struct bcm2835_dma_cb *control_block, 217 size_t len, 218 size_t period_len, 219 size_t *total_len, 220 u32 finalextrainfo) 221 { 222 size_t max_len = bcm2835_dma_max_frame_length(chan); 223 224 /* set the length taking lite-channel limitations into account */ 225 control_block->length = min_t(u32, len, max_len); 226 227 /* finished if we have no period_length */ 228 if (!period_len) 229 return; 230 231 /* 232 * period_len means: that we need to generate 233 * transfers that are terminating at every 234 * multiple of period_len - this is typically 235 * used to set the interrupt flag in info 236 * which is required during cyclic transfers 237 */ 238 239 /* have we filled in period_length yet? */ 240 if (*total_len + control_block->length < period_len) { 241 /* update number of bytes in this period so far */ 242 *total_len += control_block->length; 243 return; 244 } 245 246 /* calculate the length that remains to reach period_length */ 247 control_block->length = period_len - *total_len; 248 249 /* reset total_length for next period */ 250 *total_len = 0; 251 252 /* add extrainfo bits in info */ 253 control_block->info |= finalextrainfo; 254 } 255 256 static inline size_t bcm2835_dma_count_frames_for_sg( 257 struct bcm2835_chan *c, 258 struct scatterlist *sgl, 259 unsigned int sg_len) 260 { 261 size_t frames = 0; 262 struct scatterlist *sgent; 263 unsigned int i; 264 size_t plength = bcm2835_dma_max_frame_length(c); 265 266 for_each_sg(sgl, sgent, sg_len, i) 267 frames += bcm2835_dma_frames_for_length( 268 sg_dma_len(sgent), plength); 269 270 return frames; 271 } 272 273 /** 274 * bcm2835_dma_create_cb_chain - create a control block and fills data in 275 * 276 * @chan: the @dma_chan for which we run this 277 * @direction: the direction in which we transfer 278 * @cyclic: it is a cyclic transfer 279 * @info: the default info bits to apply per controlblock 280 * @frames: number of controlblocks to allocate 281 * @src: the src address to assign (if the S_INC bit is set 282 * in @info, then it gets incremented) 283 * @dst: the dst address to assign (if the D_INC bit is set 284 * in @info, then it gets incremented) 285 * @buf_len: the full buffer length (may also be 0) 286 * @period_len: the period length when to apply @finalextrainfo 287 * in addition to the last transfer 288 * this will also break some control-blocks early 289 * @finalextrainfo: additional bits in last controlblock 290 * (or when period_len is reached in case of cyclic) 291 * @gfp: the GFP flag to use for allocation 292 */ 293 static struct bcm2835_desc *bcm2835_dma_create_cb_chain( 294 struct dma_chan *chan, enum dma_transfer_direction direction, 295 bool cyclic, u32 info, u32 finalextrainfo, size_t frames, 296 dma_addr_t src, dma_addr_t dst, size_t buf_len, 297 size_t period_len, gfp_t gfp) 298 { 299 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 300 size_t len = buf_len, total_len; 301 size_t frame; 302 struct bcm2835_desc *d; 303 struct bcm2835_cb_entry *cb_entry; 304 struct bcm2835_dma_cb *control_block; 305 306 if (!frames) 307 return NULL; 308 309 /* allocate and setup the descriptor. */ 310 d = kzalloc(struct_size(d, cb_list, frames), gfp); 311 if (!d) 312 return NULL; 313 314 d->c = c; 315 d->dir = direction; 316 d->cyclic = cyclic; 317 318 /* 319 * Iterate over all frames, create a control block 320 * for each frame and link them together. 321 */ 322 for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) { 323 cb_entry = &d->cb_list[frame]; 324 cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp, 325 &cb_entry->paddr); 326 if (!cb_entry->cb) 327 goto error_cb; 328 329 /* fill in the control block */ 330 control_block = cb_entry->cb; 331 control_block->info = info; 332 control_block->src = src; 333 control_block->dst = dst; 334 control_block->stride = 0; 335 control_block->next = 0; 336 /* set up length in control_block if requested */ 337 if (buf_len) { 338 /* calculate length honoring period_length */ 339 bcm2835_dma_create_cb_set_length( 340 c, control_block, 341 len, period_len, &total_len, 342 cyclic ? finalextrainfo : 0); 343 344 /* calculate new remaining length */ 345 len -= control_block->length; 346 } 347 348 /* link this the last controlblock */ 349 if (frame) 350 d->cb_list[frame - 1].cb->next = cb_entry->paddr; 351 352 /* update src and dst and length */ 353 if (src && (info & BCM2835_DMA_S_INC)) 354 src += control_block->length; 355 if (dst && (info & BCM2835_DMA_D_INC)) 356 dst += control_block->length; 357 358 /* Length of total transfer */ 359 d->size += control_block->length; 360 } 361 362 /* the last frame requires extra flags */ 363 d->cb_list[d->frames - 1].cb->info |= finalextrainfo; 364 365 /* detect a size missmatch */ 366 if (buf_len && (d->size != buf_len)) 367 goto error_cb; 368 369 return d; 370 error_cb: 371 bcm2835_dma_free_cb_chain(d); 372 373 return NULL; 374 } 375 376 static void bcm2835_dma_fill_cb_chain_with_sg( 377 struct dma_chan *chan, 378 enum dma_transfer_direction direction, 379 struct bcm2835_cb_entry *cb, 380 struct scatterlist *sgl, 381 unsigned int sg_len) 382 { 383 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 384 size_t len, max_len; 385 unsigned int i; 386 dma_addr_t addr; 387 struct scatterlist *sgent; 388 389 max_len = bcm2835_dma_max_frame_length(c); 390 for_each_sg(sgl, sgent, sg_len, i) { 391 for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent); 392 len > 0; 393 addr += cb->cb->length, len -= cb->cb->length, cb++) { 394 if (direction == DMA_DEV_TO_MEM) 395 cb->cb->dst = addr; 396 else 397 cb->cb->src = addr; 398 cb->cb->length = min(len, max_len); 399 } 400 } 401 } 402 403 static void bcm2835_dma_abort(struct bcm2835_chan *c) 404 { 405 void __iomem *chan_base = c->chan_base; 406 long int timeout = 10000; 407 408 /* 409 * A zero control block address means the channel is idle. 410 * (The ACTIVE flag in the CS register is not a reliable indicator.) 411 */ 412 if (!readl(chan_base + BCM2835_DMA_ADDR)) 413 return; 414 415 /* Write 0 to the active bit - Pause the DMA */ 416 writel(0, chan_base + BCM2835_DMA_CS); 417 418 /* Wait for any current AXI transfer to complete */ 419 while ((readl(chan_base + BCM2835_DMA_CS) & 420 BCM2835_DMA_WAITING_FOR_WRITES) && --timeout) 421 cpu_relax(); 422 423 /* Peripheral might be stuck and fail to signal AXI write responses */ 424 if (!timeout) 425 dev_err(c->vc.chan.device->dev, 426 "failed to complete outstanding writes\n"); 427 428 writel(BCM2835_DMA_RESET, chan_base + BCM2835_DMA_CS); 429 } 430 431 static void bcm2835_dma_start_desc(struct bcm2835_chan *c) 432 { 433 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 434 struct bcm2835_desc *d; 435 436 if (!vd) { 437 c->desc = NULL; 438 return; 439 } 440 441 list_del(&vd->node); 442 443 c->desc = d = to_bcm2835_dma_desc(&vd->tx); 444 445 writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR); 446 writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS); 447 } 448 449 static irqreturn_t bcm2835_dma_callback(int irq, void *data) 450 { 451 struct bcm2835_chan *c = data; 452 struct bcm2835_desc *d; 453 unsigned long flags; 454 455 /* check the shared interrupt */ 456 if (c->irq_flags & IRQF_SHARED) { 457 /* check if the interrupt is enabled */ 458 flags = readl(c->chan_base + BCM2835_DMA_CS); 459 /* if not set then we are not the reason for the irq */ 460 if (!(flags & BCM2835_DMA_INT)) 461 return IRQ_NONE; 462 } 463 464 spin_lock_irqsave(&c->vc.lock, flags); 465 466 /* 467 * Clear the INT flag to receive further interrupts. Keep the channel 468 * active in case the descriptor is cyclic or in case the client has 469 * already terminated the descriptor and issued a new one. (May happen 470 * if this IRQ handler is threaded.) If the channel is finished, it 471 * will remain idle despite the ACTIVE flag being set. 472 */ 473 writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE, 474 c->chan_base + BCM2835_DMA_CS); 475 476 d = c->desc; 477 478 if (d) { 479 if (d->cyclic) { 480 /* call the cyclic callback */ 481 vchan_cyclic_callback(&d->vd); 482 } else if (!readl(c->chan_base + BCM2835_DMA_ADDR)) { 483 vchan_cookie_complete(&c->desc->vd); 484 bcm2835_dma_start_desc(c); 485 } 486 } 487 488 spin_unlock_irqrestore(&c->vc.lock, flags); 489 490 return IRQ_HANDLED; 491 } 492 493 static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) 494 { 495 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 496 struct device *dev = c->vc.chan.device->dev; 497 498 dev_dbg(dev, "Allocating DMA channel %d\n", c->ch); 499 500 /* 501 * Control blocks are 256 bit in length and must start at a 256 bit 502 * (32 byte) aligned address (BCM2835 ARM Peripherals, sec. 4.2.1.1). 503 */ 504 c->cb_pool = dma_pool_create(dev_name(dev), dev, 505 sizeof(struct bcm2835_dma_cb), 32, 0); 506 if (!c->cb_pool) { 507 dev_err(dev, "unable to allocate descriptor pool\n"); 508 return -ENOMEM; 509 } 510 511 return request_irq(c->irq_number, bcm2835_dma_callback, 512 c->irq_flags, "DMA IRQ", c); 513 } 514 515 static void bcm2835_dma_free_chan_resources(struct dma_chan *chan) 516 { 517 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 518 519 vchan_free_chan_resources(&c->vc); 520 free_irq(c->irq_number, c); 521 dma_pool_destroy(c->cb_pool); 522 523 dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch); 524 } 525 526 static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d) 527 { 528 return d->size; 529 } 530 531 static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr) 532 { 533 unsigned int i; 534 size_t size; 535 536 for (size = i = 0; i < d->frames; i++) { 537 struct bcm2835_dma_cb *control_block = d->cb_list[i].cb; 538 size_t this_size = control_block->length; 539 dma_addr_t dma; 540 541 if (d->dir == DMA_DEV_TO_MEM) 542 dma = control_block->dst; 543 else 544 dma = control_block->src; 545 546 if (size) 547 size += this_size; 548 else if (addr >= dma && addr < dma + this_size) 549 size += dma + this_size - addr; 550 } 551 552 return size; 553 } 554 555 static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, 556 dma_cookie_t cookie, struct dma_tx_state *txstate) 557 { 558 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 559 struct virt_dma_desc *vd; 560 enum dma_status ret; 561 unsigned long flags; 562 563 ret = dma_cookie_status(chan, cookie, txstate); 564 if (ret == DMA_COMPLETE || !txstate) 565 return ret; 566 567 spin_lock_irqsave(&c->vc.lock, flags); 568 vd = vchan_find_desc(&c->vc, cookie); 569 if (vd) { 570 txstate->residue = 571 bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx)); 572 } else if (c->desc && c->desc->vd.tx.cookie == cookie) { 573 struct bcm2835_desc *d = c->desc; 574 dma_addr_t pos; 575 576 if (d->dir == DMA_MEM_TO_DEV) 577 pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD); 578 else if (d->dir == DMA_DEV_TO_MEM) 579 pos = readl(c->chan_base + BCM2835_DMA_DEST_AD); 580 else 581 pos = 0; 582 583 txstate->residue = bcm2835_dma_desc_size_pos(d, pos); 584 } else { 585 txstate->residue = 0; 586 } 587 588 spin_unlock_irqrestore(&c->vc.lock, flags); 589 590 return ret; 591 } 592 593 static void bcm2835_dma_issue_pending(struct dma_chan *chan) 594 { 595 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 596 unsigned long flags; 597 598 spin_lock_irqsave(&c->vc.lock, flags); 599 if (vchan_issue_pending(&c->vc) && !c->desc) 600 bcm2835_dma_start_desc(c); 601 602 spin_unlock_irqrestore(&c->vc.lock, flags); 603 } 604 605 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( 606 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, 607 size_t len, unsigned long flags) 608 { 609 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 610 struct bcm2835_desc *d; 611 u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC; 612 u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP; 613 size_t max_len = bcm2835_dma_max_frame_length(c); 614 size_t frames; 615 616 /* if src, dst or len is not given return with an error */ 617 if (!src || !dst || !len) 618 return NULL; 619 620 /* calculate number of frames */ 621 frames = bcm2835_dma_frames_for_length(len, max_len); 622 623 /* allocate the CB chain - this also fills in the pointers */ 624 d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false, 625 info, extra, frames, 626 src, dst, len, 0, GFP_KERNEL); 627 if (!d) 628 return NULL; 629 630 return vchan_tx_prep(&c->vc, &d->vd, flags); 631 } 632 633 static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( 634 struct dma_chan *chan, 635 struct scatterlist *sgl, unsigned int sg_len, 636 enum dma_transfer_direction direction, 637 unsigned long flags, void *context) 638 { 639 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 640 struct bcm2835_desc *d; 641 dma_addr_t src = 0, dst = 0; 642 u32 info = BCM2835_DMA_WAIT_RESP; 643 u32 extra = BCM2835_DMA_INT_EN; 644 size_t frames; 645 646 if (!is_slave_direction(direction)) { 647 dev_err(chan->device->dev, 648 "%s: bad direction?\n", __func__); 649 return NULL; 650 } 651 652 if (c->dreq != 0) 653 info |= BCM2835_DMA_PER_MAP(c->dreq); 654 655 if (direction == DMA_DEV_TO_MEM) { 656 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 657 return NULL; 658 src = c->cfg.src_addr; 659 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; 660 } else { 661 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 662 return NULL; 663 dst = c->cfg.dst_addr; 664 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; 665 } 666 667 /* count frames in sg list */ 668 frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len); 669 670 /* allocate the CB chain */ 671 d = bcm2835_dma_create_cb_chain(chan, direction, false, 672 info, extra, 673 frames, src, dst, 0, 0, 674 GFP_NOWAIT); 675 if (!d) 676 return NULL; 677 678 /* fill in frames with scatterlist pointers */ 679 bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list, 680 sgl, sg_len); 681 682 return vchan_tx_prep(&c->vc, &d->vd, flags); 683 } 684 685 static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( 686 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 687 size_t period_len, enum dma_transfer_direction direction, 688 unsigned long flags) 689 { 690 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 691 struct bcm2835_desc *d; 692 dma_addr_t src, dst; 693 u32 info = BCM2835_DMA_WAIT_RESP; 694 u32 extra = BCM2835_DMA_INT_EN; 695 size_t max_len = bcm2835_dma_max_frame_length(c); 696 size_t frames; 697 698 /* Grab configuration */ 699 if (!is_slave_direction(direction)) { 700 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 701 return NULL; 702 } 703 704 if (!buf_len) { 705 dev_err(chan->device->dev, 706 "%s: bad buffer length (= 0)\n", __func__); 707 return NULL; 708 } 709 710 /* 711 * warn if buf_len is not a multiple of period_len - this may leed 712 * to unexpected latencies for interrupts and thus audiable clicks 713 */ 714 if (buf_len % period_len) 715 dev_warn_once(chan->device->dev, 716 "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n", 717 __func__, buf_len, period_len); 718 719 /* Setup DREQ channel */ 720 if (c->dreq != 0) 721 info |= BCM2835_DMA_PER_MAP(c->dreq); 722 723 if (direction == DMA_DEV_TO_MEM) { 724 if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 725 return NULL; 726 src = c->cfg.src_addr; 727 dst = buf_addr; 728 info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; 729 } else { 730 if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) 731 return NULL; 732 dst = c->cfg.dst_addr; 733 src = buf_addr; 734 info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; 735 } 736 737 /* calculate number of frames */ 738 frames = /* number of periods */ 739 DIV_ROUND_UP(buf_len, period_len) * 740 /* number of frames per period */ 741 bcm2835_dma_frames_for_length(period_len, max_len); 742 743 /* 744 * allocate the CB chain 745 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine 746 * implementation calls prep_dma_cyclic with interrupts disabled. 747 */ 748 d = bcm2835_dma_create_cb_chain(chan, direction, true, 749 info, extra, 750 frames, src, dst, buf_len, 751 period_len, GFP_NOWAIT); 752 if (!d) 753 return NULL; 754 755 /* wrap around into a loop */ 756 d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr; 757 758 return vchan_tx_prep(&c->vc, &d->vd, flags); 759 } 760 761 static int bcm2835_dma_slave_config(struct dma_chan *chan, 762 struct dma_slave_config *cfg) 763 { 764 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 765 766 c->cfg = *cfg; 767 768 return 0; 769 } 770 771 static int bcm2835_dma_terminate_all(struct dma_chan *chan) 772 { 773 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 774 unsigned long flags; 775 LIST_HEAD(head); 776 777 spin_lock_irqsave(&c->vc.lock, flags); 778 779 /* stop DMA activity */ 780 if (c->desc) { 781 vchan_terminate_vdesc(&c->desc->vd); 782 c->desc = NULL; 783 bcm2835_dma_abort(c); 784 } 785 786 vchan_get_all_descriptors(&c->vc, &head); 787 spin_unlock_irqrestore(&c->vc.lock, flags); 788 vchan_dma_desc_free_list(&c->vc, &head); 789 790 return 0; 791 } 792 793 static void bcm2835_dma_synchronize(struct dma_chan *chan) 794 { 795 struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); 796 797 vchan_synchronize(&c->vc); 798 } 799 800 static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, 801 int irq, unsigned int irq_flags) 802 { 803 struct bcm2835_chan *c; 804 805 c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL); 806 if (!c) 807 return -ENOMEM; 808 809 c->vc.desc_free = bcm2835_dma_desc_free; 810 vchan_init(&c->vc, &d->ddev); 811 812 c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); 813 c->ch = chan_id; 814 c->irq_number = irq; 815 c->irq_flags = irq_flags; 816 817 /* check in DEBUG register if this is a LITE channel */ 818 if (readl(c->chan_base + BCM2835_DMA_DEBUG) & 819 BCM2835_DMA_DEBUG_LITE) 820 c->is_lite_channel = true; 821 822 return 0; 823 } 824 825 static void bcm2835_dma_free(struct bcm2835_dmadev *od) 826 { 827 struct bcm2835_chan *c, *next; 828 829 list_for_each_entry_safe(c, next, &od->ddev.channels, 830 vc.chan.device_node) { 831 list_del(&c->vc.chan.device_node); 832 tasklet_kill(&c->vc.task); 833 } 834 } 835 836 static const struct of_device_id bcm2835_dma_of_match[] = { 837 { .compatible = "brcm,bcm2835-dma", }, 838 {}, 839 }; 840 MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match); 841 842 static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, 843 struct of_dma *ofdma) 844 { 845 struct bcm2835_dmadev *d = ofdma->of_dma_data; 846 struct dma_chan *chan; 847 848 chan = dma_get_any_slave_channel(&d->ddev); 849 if (!chan) 850 return NULL; 851 852 /* Set DREQ from param */ 853 to_bcm2835_dma_chan(chan)->dreq = spec->args[0]; 854 855 return chan; 856 } 857 858 static int bcm2835_dma_probe(struct platform_device *pdev) 859 { 860 struct bcm2835_dmadev *od; 861 struct resource *res; 862 void __iomem *base; 863 int rc; 864 int i, j; 865 int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1]; 866 int irq_flags; 867 uint32_t chans_available; 868 char chan_name[BCM2835_DMA_CHAN_NAME_SIZE]; 869 870 if (!pdev->dev.dma_mask) 871 pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; 872 873 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 874 if (rc) 875 return rc; 876 877 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); 878 if (!od) 879 return -ENOMEM; 880 881 pdev->dev.dma_parms = &od->dma_parms; 882 dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF); 883 884 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 885 base = devm_ioremap_resource(&pdev->dev, res); 886 if (IS_ERR(base)) 887 return PTR_ERR(base); 888 889 od->base = base; 890 891 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 892 dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask); 893 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); 894 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); 895 od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources; 896 od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources; 897 od->ddev.device_tx_status = bcm2835_dma_tx_status; 898 od->ddev.device_issue_pending = bcm2835_dma_issue_pending; 899 od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic; 900 od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg; 901 od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy; 902 od->ddev.device_config = bcm2835_dma_slave_config; 903 od->ddev.device_terminate_all = bcm2835_dma_terminate_all; 904 od->ddev.device_synchronize = bcm2835_dma_synchronize; 905 od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 906 od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 907 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | 908 BIT(DMA_MEM_TO_MEM); 909 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 910 od->ddev.dev = &pdev->dev; 911 INIT_LIST_HEAD(&od->ddev.channels); 912 913 platform_set_drvdata(pdev, od); 914 915 /* Request DMA channel mask from device tree */ 916 if (of_property_read_u32(pdev->dev.of_node, 917 "brcm,dma-channel-mask", 918 &chans_available)) { 919 dev_err(&pdev->dev, "Failed to get channel mask\n"); 920 rc = -EINVAL; 921 goto err_no_dma; 922 } 923 924 /* get irqs for each channel that we support */ 925 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { 926 /* skip masked out channels */ 927 if (!(chans_available & (1 << i))) { 928 irq[i] = -1; 929 continue; 930 } 931 932 /* get the named irq */ 933 snprintf(chan_name, sizeof(chan_name), "dma%i", i); 934 irq[i] = platform_get_irq_byname(pdev, chan_name); 935 if (irq[i] >= 0) 936 continue; 937 938 /* legacy device tree case handling */ 939 dev_warn_once(&pdev->dev, 940 "missing interrupt-names property in device tree - legacy interpretation is used\n"); 941 /* 942 * in case of channel >= 11 943 * use the 11th interrupt and that is shared 944 */ 945 irq[i] = platform_get_irq(pdev, i < 11 ? i : 11); 946 } 947 948 /* get irqs for each channel */ 949 for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { 950 /* skip channels without irq */ 951 if (irq[i] < 0) 952 continue; 953 954 /* check if there are other channels that also use this irq */ 955 irq_flags = 0; 956 for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++) 957 if ((i != j) && (irq[j] == irq[i])) { 958 irq_flags = IRQF_SHARED; 959 break; 960 } 961 962 /* initialize the channel */ 963 rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags); 964 if (rc) 965 goto err_no_dma; 966 } 967 968 dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i); 969 970 /* Device-tree DMA controller registration */ 971 rc = of_dma_controller_register(pdev->dev.of_node, 972 bcm2835_dma_xlate, od); 973 if (rc) { 974 dev_err(&pdev->dev, "Failed to register DMA controller\n"); 975 goto err_no_dma; 976 } 977 978 rc = dma_async_device_register(&od->ddev); 979 if (rc) { 980 dev_err(&pdev->dev, 981 "Failed to register slave DMA engine device: %d\n", rc); 982 goto err_no_dma; 983 } 984 985 dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n"); 986 987 return 0; 988 989 err_no_dma: 990 bcm2835_dma_free(od); 991 return rc; 992 } 993 994 static int bcm2835_dma_remove(struct platform_device *pdev) 995 { 996 struct bcm2835_dmadev *od = platform_get_drvdata(pdev); 997 998 dma_async_device_unregister(&od->ddev); 999 bcm2835_dma_free(od); 1000 1001 return 0; 1002 } 1003 1004 static struct platform_driver bcm2835_dma_driver = { 1005 .probe = bcm2835_dma_probe, 1006 .remove = bcm2835_dma_remove, 1007 .driver = { 1008 .name = "bcm2835-dma", 1009 .of_match_table = of_match_ptr(bcm2835_dma_of_match), 1010 }, 1011 }; 1012 1013 module_platform_driver(bcm2835_dma_driver); 1014 1015 MODULE_ALIAS("platform:bcm2835-dma"); 1016 MODULE_DESCRIPTION("BCM2835 DMA engine driver"); 1017 MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); 1018 MODULE_LICENSE("GPL"); 1019