1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Renesas R-Car Gen2 DMA Controller Driver 4 * 5 * Copyright (C) 2014 Renesas Electronics Inc. 6 * 7 * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> 8 */ 9 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmaengine.h> 13 #include <linux/interrupt.h> 14 #include <linux/list.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/of.h> 18 #include <linux/of_dma.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_runtime.h> 22 #include <linux/slab.h> 23 #include <linux/spinlock.h> 24 25 #include "../dmaengine.h" 26 27 /* 28 * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer 29 * @node: entry in the parent's chunks list 30 * @src_addr: device source address 31 * @dst_addr: device destination address 32 * @size: transfer size in bytes 33 */ 34 struct rcar_dmac_xfer_chunk { 35 struct list_head node; 36 37 dma_addr_t src_addr; 38 dma_addr_t dst_addr; 39 u32 size; 40 }; 41 42 /* 43 * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk 44 * @sar: value of the SAR register (source address) 45 * @dar: value of the DAR register (destination address) 46 * @tcr: value of the TCR register (transfer count) 47 */ 48 struct rcar_dmac_hw_desc { 49 u32 sar; 50 u32 dar; 51 u32 tcr; 52 u32 reserved; 53 } __attribute__((__packed__)); 54 55 /* 56 * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor 57 * @async_tx: base DMA asynchronous transaction descriptor 58 * @direction: direction of the DMA transfer 59 * @xfer_shift: log2 of the transfer size 60 * @chcr: value of the channel configuration register for this transfer 61 * @node: entry in the channel's descriptors lists 62 * @chunks: list of transfer chunks for this transfer 63 * @running: the transfer chunk being currently processed 64 * @nchunks: number of transfer chunks for this transfer 65 * @hwdescs.use: whether the transfer descriptor uses hardware descriptors 66 * @hwdescs.mem: hardware descriptors memory for the transfer 67 * @hwdescs.dma: device address of the hardware descriptors memory 68 * @hwdescs.size: size of the hardware descriptors in bytes 69 * @size: transfer size in bytes 70 * @cyclic: when set indicates that the DMA transfer is cyclic 71 */ 72 struct rcar_dmac_desc { 73 struct dma_async_tx_descriptor async_tx; 74 enum dma_transfer_direction direction; 75 unsigned int xfer_shift; 76 u32 chcr; 77 78 struct list_head node; 79 struct list_head chunks; 80 struct rcar_dmac_xfer_chunk *running; 81 unsigned int nchunks; 82 83 struct { 84 bool use; 85 struct rcar_dmac_hw_desc *mem; 86 dma_addr_t dma; 87 size_t size; 88 } hwdescs; 89 90 unsigned int size; 91 bool cyclic; 92 }; 93 94 #define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx) 95 96 /* 97 * struct rcar_dmac_desc_page - One page worth of descriptors 98 * @node: entry in the channel's pages list 99 * @descs: array of DMA descriptors 100 * @chunks: array of transfer chunk descriptors 101 */ 102 struct rcar_dmac_desc_page { 103 struct list_head node; 104 105 union { 106 struct rcar_dmac_desc descs[0]; 107 struct rcar_dmac_xfer_chunk chunks[0]; 108 }; 109 }; 110 111 #define RCAR_DMAC_DESCS_PER_PAGE \ 112 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \ 113 sizeof(struct rcar_dmac_desc)) 114 #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \ 115 ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \ 116 sizeof(struct rcar_dmac_xfer_chunk)) 117 118 /* 119 * struct rcar_dmac_chan_slave - Slave configuration 120 * @slave_addr: slave memory address 121 * @xfer_size: size (in bytes) of hardware transfers 122 */ 123 struct rcar_dmac_chan_slave { 124 phys_addr_t slave_addr; 125 unsigned int xfer_size; 126 }; 127 128 /* 129 * struct rcar_dmac_chan_map - Map of slave device phys to dma address 130 * @addr: slave dma address 131 * @dir: direction of mapping 132 * @slave: slave configuration that is mapped 133 */ 134 struct rcar_dmac_chan_map { 135 dma_addr_t addr; 136 enum dma_data_direction dir; 137 struct rcar_dmac_chan_slave slave; 138 }; 139 140 /* 141 * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel 142 * @chan: base DMA channel object 143 * @iomem: channel I/O memory base 144 * @index: index of this channel in the controller 145 * @irq: channel IRQ 146 * @src: slave memory address and size on the source side 147 * @dst: slave memory address and size on the destination side 148 * @mid_rid: hardware MID/RID for the DMA client using this channel 149 * @lock: protects the channel CHCR register and the desc members 150 * @desc.free: list of free descriptors 151 * @desc.pending: list of pending descriptors (submitted with tx_submit) 152 * @desc.active: list of active descriptors (activated with issue_pending) 153 * @desc.done: list of completed descriptors 154 * @desc.wait: list of descriptors waiting for an ack 155 * @desc.running: the descriptor being processed (a member of the active list) 156 * @desc.chunks_free: list of free transfer chunk descriptors 157 * @desc.pages: list of pages used by allocated descriptors 158 */ 159 struct rcar_dmac_chan { 160 struct dma_chan chan; 161 void __iomem *iomem; 162 unsigned int index; 163 int irq; 164 165 struct rcar_dmac_chan_slave src; 166 struct rcar_dmac_chan_slave dst; 167 struct rcar_dmac_chan_map map; 168 int mid_rid; 169 170 spinlock_t lock; 171 172 struct { 173 struct list_head free; 174 struct list_head pending; 175 struct list_head active; 176 struct list_head done; 177 struct list_head wait; 178 struct rcar_dmac_desc *running; 179 180 struct list_head chunks_free; 181 182 struct list_head pages; 183 } desc; 184 }; 185 186 #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan) 187 188 /* 189 * struct rcar_dmac - R-Car Gen2 DMA Controller 190 * @engine: base DMA engine object 191 * @dev: the hardware device 192 * @iomem: remapped I/O memory base 193 * @n_channels: number of available channels 194 * @channels: array of DMAC channels 195 * @modules: bitmask of client modules in use 196 */ 197 struct rcar_dmac { 198 struct dma_device engine; 199 struct device *dev; 200 void __iomem *iomem; 201 202 unsigned int n_channels; 203 struct rcar_dmac_chan *channels; 204 205 DECLARE_BITMAP(modules, 256); 206 }; 207 208 #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) 209 210 /* ----------------------------------------------------------------------------- 211 * Registers 212 */ 213 214 #define RCAR_DMAC_CHAN_OFFSET(i) (0x8000 + 0x80 * (i)) 215 216 #define RCAR_DMAISTA 0x0020 217 #define RCAR_DMASEC 0x0030 218 #define RCAR_DMAOR 0x0060 219 #define RCAR_DMAOR_PRI_FIXED (0 << 8) 220 #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) 221 #define RCAR_DMAOR_AE (1 << 2) 222 #define RCAR_DMAOR_DME (1 << 0) 223 #define RCAR_DMACHCLR 0x0080 224 #define RCAR_DMADPSEC 0x00a0 225 226 #define RCAR_DMASAR 0x0000 227 #define RCAR_DMADAR 0x0004 228 #define RCAR_DMATCR 0x0008 229 #define RCAR_DMATCR_MASK 0x00ffffff 230 #define RCAR_DMATSR 0x0028 231 #define RCAR_DMACHCR 0x000c 232 #define RCAR_DMACHCR_CAE (1 << 31) 233 #define RCAR_DMACHCR_CAIE (1 << 30) 234 #define RCAR_DMACHCR_DPM_DISABLED (0 << 28) 235 #define RCAR_DMACHCR_DPM_ENABLED (1 << 28) 236 #define RCAR_DMACHCR_DPM_REPEAT (2 << 28) 237 #define RCAR_DMACHCR_DPM_INFINITE (3 << 28) 238 #define RCAR_DMACHCR_RPT_SAR (1 << 27) 239 #define RCAR_DMACHCR_RPT_DAR (1 << 26) 240 #define RCAR_DMACHCR_RPT_TCR (1 << 25) 241 #define RCAR_DMACHCR_DPB (1 << 22) 242 #define RCAR_DMACHCR_DSE (1 << 19) 243 #define RCAR_DMACHCR_DSIE (1 << 18) 244 #define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3)) 245 #define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3)) 246 #define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3)) 247 #define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3)) 248 #define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3)) 249 #define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3)) 250 #define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3)) 251 #define RCAR_DMACHCR_DM_FIXED (0 << 14) 252 #define RCAR_DMACHCR_DM_INC (1 << 14) 253 #define RCAR_DMACHCR_DM_DEC (2 << 14) 254 #define RCAR_DMACHCR_SM_FIXED (0 << 12) 255 #define RCAR_DMACHCR_SM_INC (1 << 12) 256 #define RCAR_DMACHCR_SM_DEC (2 << 12) 257 #define RCAR_DMACHCR_RS_AUTO (4 << 8) 258 #define RCAR_DMACHCR_RS_DMARS (8 << 8) 259 #define RCAR_DMACHCR_IE (1 << 2) 260 #define RCAR_DMACHCR_TE (1 << 1) 261 #define RCAR_DMACHCR_DE (1 << 0) 262 #define RCAR_DMATCRB 0x0018 263 #define RCAR_DMATSRB 0x0038 264 #define RCAR_DMACHCRB 0x001c 265 #define RCAR_DMACHCRB_DCNT(n) ((n) << 24) 266 #define RCAR_DMACHCRB_DPTR_MASK (0xff << 16) 267 #define RCAR_DMACHCRB_DPTR_SHIFT 16 268 #define RCAR_DMACHCRB_DRST (1 << 15) 269 #define RCAR_DMACHCRB_DTS (1 << 8) 270 #define RCAR_DMACHCRB_SLM_NORMAL (0 << 4) 271 #define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4) 272 #define RCAR_DMACHCRB_PRI(n) ((n) << 0) 273 #define RCAR_DMARS 0x0040 274 #define RCAR_DMABUFCR 0x0048 275 #define RCAR_DMABUFCR_MBU(n) ((n) << 16) 276 #define RCAR_DMABUFCR_ULB(n) ((n) << 0) 277 #define RCAR_DMADPBASE 0x0050 278 #define RCAR_DMADPBASE_MASK 0xfffffff0 279 #define RCAR_DMADPBASE_SEL (1 << 0) 280 #define RCAR_DMADPCR 0x0054 281 #define RCAR_DMADPCR_DIPT(n) ((n) << 24) 282 #define RCAR_DMAFIXSAR 0x0010 283 #define RCAR_DMAFIXDAR 0x0014 284 #define RCAR_DMAFIXDPBASE 0x0060 285 286 /* Hardcode the MEMCPY transfer size to 4 bytes. */ 287 #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 288 289 /* ----------------------------------------------------------------------------- 290 * Device access 291 */ 292 293 static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) 294 { 295 if (reg == RCAR_DMAOR) 296 writew(data, dmac->iomem + reg); 297 else 298 writel(data, dmac->iomem + reg); 299 } 300 301 static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) 302 { 303 if (reg == RCAR_DMAOR) 304 return readw(dmac->iomem + reg); 305 else 306 return readl(dmac->iomem + reg); 307 } 308 309 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) 310 { 311 if (reg == RCAR_DMARS) 312 return readw(chan->iomem + reg); 313 else 314 return readl(chan->iomem + reg); 315 } 316 317 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) 318 { 319 if (reg == RCAR_DMARS) 320 writew(data, chan->iomem + reg); 321 else 322 writel(data, chan->iomem + reg); 323 } 324 325 /* ----------------------------------------------------------------------------- 326 * Initialization and configuration 327 */ 328 329 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) 330 { 331 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 332 333 return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)); 334 } 335 336 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) 337 { 338 struct rcar_dmac_desc *desc = chan->desc.running; 339 u32 chcr = desc->chcr; 340 341 WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan)); 342 343 if (chan->mid_rid >= 0) 344 rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid); 345 346 if (desc->hwdescs.use) { 347 struct rcar_dmac_xfer_chunk *chunk = 348 list_first_entry(&desc->chunks, 349 struct rcar_dmac_xfer_chunk, node); 350 351 dev_dbg(chan->chan.device->dev, 352 "chan%u: queue desc %p: %u@%pad\n", 353 chan->index, desc, desc->nchunks, &desc->hwdescs.dma); 354 355 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 356 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 357 chunk->src_addr >> 32); 358 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 359 chunk->dst_addr >> 32); 360 rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE, 361 desc->hwdescs.dma >> 32); 362 #endif 363 rcar_dmac_chan_write(chan, RCAR_DMADPBASE, 364 (desc->hwdescs.dma & 0xfffffff0) | 365 RCAR_DMADPBASE_SEL); 366 rcar_dmac_chan_write(chan, RCAR_DMACHCRB, 367 RCAR_DMACHCRB_DCNT(desc->nchunks - 1) | 368 RCAR_DMACHCRB_DRST); 369 370 /* 371 * Errata: When descriptor memory is accessed through an IOMMU 372 * the DMADAR register isn't initialized automatically from the 373 * first descriptor at beginning of transfer by the DMAC like it 374 * should. Initialize it manually with the destination address 375 * of the first chunk. 376 */ 377 rcar_dmac_chan_write(chan, RCAR_DMADAR, 378 chunk->dst_addr & 0xffffffff); 379 380 /* 381 * Program the descriptor stage interrupt to occur after the end 382 * of the first stage. 383 */ 384 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1)); 385 386 chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR 387 | RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB; 388 389 /* 390 * If the descriptor isn't cyclic enable normal descriptor mode 391 * and the transfer completion interrupt. 392 */ 393 if (!desc->cyclic) 394 chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE; 395 /* 396 * If the descriptor is cyclic and has a callback enable the 397 * descriptor stage interrupt in infinite repeat mode. 398 */ 399 else if (desc->async_tx.callback) 400 chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE; 401 /* 402 * Otherwise just select infinite repeat mode without any 403 * interrupt. 404 */ 405 else 406 chcr |= RCAR_DMACHCR_DPM_INFINITE; 407 } else { 408 struct rcar_dmac_xfer_chunk *chunk = desc->running; 409 410 dev_dbg(chan->chan.device->dev, 411 "chan%u: queue chunk %p: %u@%pad -> %pad\n", 412 chan->index, chunk, chunk->size, &chunk->src_addr, 413 &chunk->dst_addr); 414 415 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 416 rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR, 417 chunk->src_addr >> 32); 418 rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR, 419 chunk->dst_addr >> 32); 420 #endif 421 rcar_dmac_chan_write(chan, RCAR_DMASAR, 422 chunk->src_addr & 0xffffffff); 423 rcar_dmac_chan_write(chan, RCAR_DMADAR, 424 chunk->dst_addr & 0xffffffff); 425 rcar_dmac_chan_write(chan, RCAR_DMATCR, 426 chunk->size >> desc->xfer_shift); 427 428 chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE; 429 } 430 431 rcar_dmac_chan_write(chan, RCAR_DMACHCR, 432 chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE); 433 } 434 435 static int rcar_dmac_init(struct rcar_dmac *dmac) 436 { 437 u16 dmaor; 438 439 /* Clear all channels and enable the DMAC globally. */ 440 rcar_dmac_write(dmac, RCAR_DMACHCLR, GENMASK(dmac->n_channels - 1, 0)); 441 rcar_dmac_write(dmac, RCAR_DMAOR, 442 RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); 443 444 dmaor = rcar_dmac_read(dmac, RCAR_DMAOR); 445 if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) { 446 dev_warn(dmac->dev, "DMAOR initialization failed.\n"); 447 return -EIO; 448 } 449 450 return 0; 451 } 452 453 /* ----------------------------------------------------------------------------- 454 * Descriptors submission 455 */ 456 457 static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx) 458 { 459 struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan); 460 struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx); 461 unsigned long flags; 462 dma_cookie_t cookie; 463 464 spin_lock_irqsave(&chan->lock, flags); 465 466 cookie = dma_cookie_assign(tx); 467 468 dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n", 469 chan->index, tx->cookie, desc); 470 471 list_add_tail(&desc->node, &chan->desc.pending); 472 desc->running = list_first_entry(&desc->chunks, 473 struct rcar_dmac_xfer_chunk, node); 474 475 spin_unlock_irqrestore(&chan->lock, flags); 476 477 return cookie; 478 } 479 480 /* ----------------------------------------------------------------------------- 481 * Descriptors allocation and free 482 */ 483 484 /* 485 * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors 486 * @chan: the DMA channel 487 * @gfp: allocation flags 488 */ 489 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 490 { 491 struct rcar_dmac_desc_page *page; 492 unsigned long flags; 493 LIST_HEAD(list); 494 unsigned int i; 495 496 page = (void *)get_zeroed_page(gfp); 497 if (!page) 498 return -ENOMEM; 499 500 for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) { 501 struct rcar_dmac_desc *desc = &page->descs[i]; 502 503 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 504 desc->async_tx.tx_submit = rcar_dmac_tx_submit; 505 INIT_LIST_HEAD(&desc->chunks); 506 507 list_add_tail(&desc->node, &list); 508 } 509 510 spin_lock_irqsave(&chan->lock, flags); 511 list_splice_tail(&list, &chan->desc.free); 512 list_add_tail(&page->node, &chan->desc.pages); 513 spin_unlock_irqrestore(&chan->lock, flags); 514 515 return 0; 516 } 517 518 /* 519 * rcar_dmac_desc_put - Release a DMA transfer descriptor 520 * @chan: the DMA channel 521 * @desc: the descriptor 522 * 523 * Put the descriptor and its transfer chunk descriptors back in the channel's 524 * free descriptors lists. The descriptor's chunks list will be reinitialized to 525 * an empty list as a result. 526 * 527 * The descriptor must have been removed from the channel's lists before calling 528 * this function. 529 */ 530 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, 531 struct rcar_dmac_desc *desc) 532 { 533 unsigned long flags; 534 535 spin_lock_irqsave(&chan->lock, flags); 536 list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free); 537 list_add(&desc->node, &chan->desc.free); 538 spin_unlock_irqrestore(&chan->lock, flags); 539 } 540 541 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan) 542 { 543 struct rcar_dmac_desc *desc, *_desc; 544 unsigned long flags; 545 LIST_HEAD(list); 546 547 /* 548 * We have to temporarily move all descriptors from the wait list to a 549 * local list as iterating over the wait list, even with 550 * list_for_each_entry_safe, isn't safe if we release the channel lock 551 * around the rcar_dmac_desc_put() call. 552 */ 553 spin_lock_irqsave(&chan->lock, flags); 554 list_splice_init(&chan->desc.wait, &list); 555 spin_unlock_irqrestore(&chan->lock, flags); 556 557 list_for_each_entry_safe(desc, _desc, &list, node) { 558 if (async_tx_test_ack(&desc->async_tx)) { 559 list_del(&desc->node); 560 rcar_dmac_desc_put(chan, desc); 561 } 562 } 563 564 if (list_empty(&list)) 565 return; 566 567 /* Put the remaining descriptors back in the wait list. */ 568 spin_lock_irqsave(&chan->lock, flags); 569 list_splice(&list, &chan->desc.wait); 570 spin_unlock_irqrestore(&chan->lock, flags); 571 } 572 573 /* 574 * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer 575 * @chan: the DMA channel 576 * 577 * Locking: This function must be called in a non-atomic context. 578 * 579 * Return: A pointer to the allocated descriptor or NULL if no descriptor can 580 * be allocated. 581 */ 582 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan) 583 { 584 struct rcar_dmac_desc *desc; 585 unsigned long flags; 586 int ret; 587 588 /* Recycle acked descriptors before attempting allocation. */ 589 rcar_dmac_desc_recycle_acked(chan); 590 591 spin_lock_irqsave(&chan->lock, flags); 592 593 while (list_empty(&chan->desc.free)) { 594 /* 595 * No free descriptors, allocate a page worth of them and try 596 * again, as someone else could race us to get the newly 597 * allocated descriptors. If the allocation fails return an 598 * error. 599 */ 600 spin_unlock_irqrestore(&chan->lock, flags); 601 ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT); 602 if (ret < 0) 603 return NULL; 604 spin_lock_irqsave(&chan->lock, flags); 605 } 606 607 desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node); 608 list_del(&desc->node); 609 610 spin_unlock_irqrestore(&chan->lock, flags); 611 612 return desc; 613 } 614 615 /* 616 * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks 617 * @chan: the DMA channel 618 * @gfp: allocation flags 619 */ 620 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp) 621 { 622 struct rcar_dmac_desc_page *page; 623 unsigned long flags; 624 LIST_HEAD(list); 625 unsigned int i; 626 627 page = (void *)get_zeroed_page(gfp); 628 if (!page) 629 return -ENOMEM; 630 631 for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) { 632 struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i]; 633 634 list_add_tail(&chunk->node, &list); 635 } 636 637 spin_lock_irqsave(&chan->lock, flags); 638 list_splice_tail(&list, &chan->desc.chunks_free); 639 list_add_tail(&page->node, &chan->desc.pages); 640 spin_unlock_irqrestore(&chan->lock, flags); 641 642 return 0; 643 } 644 645 /* 646 * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer 647 * @chan: the DMA channel 648 * 649 * Locking: This function must be called in a non-atomic context. 650 * 651 * Return: A pointer to the allocated transfer chunk descriptor or NULL if no 652 * descriptor can be allocated. 653 */ 654 static struct rcar_dmac_xfer_chunk * 655 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan) 656 { 657 struct rcar_dmac_xfer_chunk *chunk; 658 unsigned long flags; 659 int ret; 660 661 spin_lock_irqsave(&chan->lock, flags); 662 663 while (list_empty(&chan->desc.chunks_free)) { 664 /* 665 * No free descriptors, allocate a page worth of them and try 666 * again, as someone else could race us to get the newly 667 * allocated descriptors. If the allocation fails return an 668 * error. 669 */ 670 spin_unlock_irqrestore(&chan->lock, flags); 671 ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT); 672 if (ret < 0) 673 return NULL; 674 spin_lock_irqsave(&chan->lock, flags); 675 } 676 677 chunk = list_first_entry(&chan->desc.chunks_free, 678 struct rcar_dmac_xfer_chunk, node); 679 list_del(&chunk->node); 680 681 spin_unlock_irqrestore(&chan->lock, flags); 682 683 return chunk; 684 } 685 686 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan, 687 struct rcar_dmac_desc *desc, size_t size) 688 { 689 /* 690 * dma_alloc_coherent() allocates memory in page size increments. To 691 * avoid reallocating the hardware descriptors when the allocated size 692 * wouldn't change align the requested size to a multiple of the page 693 * size. 694 */ 695 size = PAGE_ALIGN(size); 696 697 if (desc->hwdescs.size == size) 698 return; 699 700 if (desc->hwdescs.mem) { 701 dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size, 702 desc->hwdescs.mem, desc->hwdescs.dma); 703 desc->hwdescs.mem = NULL; 704 desc->hwdescs.size = 0; 705 } 706 707 if (!size) 708 return; 709 710 desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size, 711 &desc->hwdescs.dma, GFP_NOWAIT); 712 if (!desc->hwdescs.mem) 713 return; 714 715 desc->hwdescs.size = size; 716 } 717 718 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan, 719 struct rcar_dmac_desc *desc) 720 { 721 struct rcar_dmac_xfer_chunk *chunk; 722 struct rcar_dmac_hw_desc *hwdesc; 723 724 rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc)); 725 726 hwdesc = desc->hwdescs.mem; 727 if (!hwdesc) 728 return -ENOMEM; 729 730 list_for_each_entry(chunk, &desc->chunks, node) { 731 hwdesc->sar = chunk->src_addr; 732 hwdesc->dar = chunk->dst_addr; 733 hwdesc->tcr = chunk->size >> desc->xfer_shift; 734 hwdesc++; 735 } 736 737 return 0; 738 } 739 740 /* ----------------------------------------------------------------------------- 741 * Stop and reset 742 */ 743 static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan) 744 { 745 u32 chcr; 746 unsigned int i; 747 748 /* 749 * Ensure that the setting of the DE bit is actually 0 after 750 * clearing it. 751 */ 752 for (i = 0; i < 1024; i++) { 753 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 754 if (!(chcr & RCAR_DMACHCR_DE)) 755 return; 756 udelay(1); 757 } 758 759 dev_err(chan->chan.device->dev, "CHCR DE check error\n"); 760 } 761 762 static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan) 763 { 764 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 765 766 /* set DE=0 and flush remaining data */ 767 rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE)); 768 769 /* make sure all remaining data was flushed */ 770 rcar_dmac_chcr_de_barrier(chan); 771 } 772 773 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan) 774 { 775 u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 776 777 chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE | 778 RCAR_DMACHCR_TE | RCAR_DMACHCR_DE | 779 RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE); 780 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr); 781 rcar_dmac_chcr_de_barrier(chan); 782 } 783 784 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) 785 { 786 struct rcar_dmac_desc *desc, *_desc; 787 unsigned long flags; 788 LIST_HEAD(descs); 789 790 spin_lock_irqsave(&chan->lock, flags); 791 792 /* Move all non-free descriptors to the local lists. */ 793 list_splice_init(&chan->desc.pending, &descs); 794 list_splice_init(&chan->desc.active, &descs); 795 list_splice_init(&chan->desc.done, &descs); 796 list_splice_init(&chan->desc.wait, &descs); 797 798 chan->desc.running = NULL; 799 800 spin_unlock_irqrestore(&chan->lock, flags); 801 802 list_for_each_entry_safe(desc, _desc, &descs, node) { 803 list_del(&desc->node); 804 rcar_dmac_desc_put(chan, desc); 805 } 806 } 807 808 static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) 809 { 810 unsigned int i; 811 812 /* Stop all channels. */ 813 for (i = 0; i < dmac->n_channels; ++i) { 814 struct rcar_dmac_chan *chan = &dmac->channels[i]; 815 816 /* Stop and reinitialize the channel. */ 817 spin_lock_irq(&chan->lock); 818 rcar_dmac_chan_halt(chan); 819 spin_unlock_irq(&chan->lock); 820 } 821 } 822 823 static int rcar_dmac_chan_pause(struct dma_chan *chan) 824 { 825 unsigned long flags; 826 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 827 828 spin_lock_irqsave(&rchan->lock, flags); 829 rcar_dmac_clear_chcr_de(rchan); 830 spin_unlock_irqrestore(&rchan->lock, flags); 831 832 return 0; 833 } 834 835 /* ----------------------------------------------------------------------------- 836 * Descriptors preparation 837 */ 838 839 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan, 840 struct rcar_dmac_desc *desc) 841 { 842 static const u32 chcr_ts[] = { 843 RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B, 844 RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B, 845 RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B, 846 RCAR_DMACHCR_TS_64B, 847 }; 848 849 unsigned int xfer_size; 850 u32 chcr; 851 852 switch (desc->direction) { 853 case DMA_DEV_TO_MEM: 854 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED 855 | RCAR_DMACHCR_RS_DMARS; 856 xfer_size = chan->src.xfer_size; 857 break; 858 859 case DMA_MEM_TO_DEV: 860 chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC 861 | RCAR_DMACHCR_RS_DMARS; 862 xfer_size = chan->dst.xfer_size; 863 break; 864 865 case DMA_MEM_TO_MEM: 866 default: 867 chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC 868 | RCAR_DMACHCR_RS_AUTO; 869 xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE; 870 break; 871 } 872 873 desc->xfer_shift = ilog2(xfer_size); 874 desc->chcr = chcr | chcr_ts[desc->xfer_shift]; 875 } 876 877 /* 878 * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list 879 * 880 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 881 * converted to scatter-gather to guarantee consistent locking and a correct 882 * list manipulation. For slave DMA direction carries the usual meaning, and, 883 * logically, the SG list is RAM and the addr variable contains slave address, 884 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 885 * and the SG list contains only one element and points at the source buffer. 886 */ 887 static struct dma_async_tx_descriptor * 888 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl, 889 unsigned int sg_len, dma_addr_t dev_addr, 890 enum dma_transfer_direction dir, unsigned long dma_flags, 891 bool cyclic) 892 { 893 struct rcar_dmac_xfer_chunk *chunk; 894 struct rcar_dmac_desc *desc; 895 struct scatterlist *sg; 896 unsigned int nchunks = 0; 897 unsigned int max_chunk_size; 898 unsigned int full_size = 0; 899 bool cross_boundary = false; 900 unsigned int i; 901 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 902 u32 high_dev_addr; 903 u32 high_mem_addr; 904 #endif 905 906 desc = rcar_dmac_desc_get(chan); 907 if (!desc) 908 return NULL; 909 910 desc->async_tx.flags = dma_flags; 911 desc->async_tx.cookie = -EBUSY; 912 913 desc->cyclic = cyclic; 914 desc->direction = dir; 915 916 rcar_dmac_chan_configure_desc(chan, desc); 917 918 max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift; 919 920 /* 921 * Allocate and fill the transfer chunk descriptors. We own the only 922 * reference to the DMA descriptor, there's no need for locking. 923 */ 924 for_each_sg(sgl, sg, sg_len, i) { 925 dma_addr_t mem_addr = sg_dma_address(sg); 926 unsigned int len = sg_dma_len(sg); 927 928 full_size += len; 929 930 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 931 if (i == 0) { 932 high_dev_addr = dev_addr >> 32; 933 high_mem_addr = mem_addr >> 32; 934 } 935 936 if ((dev_addr >> 32 != high_dev_addr) || 937 (mem_addr >> 32 != high_mem_addr)) 938 cross_boundary = true; 939 #endif 940 while (len) { 941 unsigned int size = min(len, max_chunk_size); 942 943 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 944 /* 945 * Prevent individual transfers from crossing 4GB 946 * boundaries. 947 */ 948 if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) { 949 size = ALIGN(dev_addr, 1ULL << 32) - dev_addr; 950 cross_boundary = true; 951 } 952 if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) { 953 size = ALIGN(mem_addr, 1ULL << 32) - mem_addr; 954 cross_boundary = true; 955 } 956 #endif 957 958 chunk = rcar_dmac_xfer_chunk_get(chan); 959 if (!chunk) { 960 rcar_dmac_desc_put(chan, desc); 961 return NULL; 962 } 963 964 if (dir == DMA_DEV_TO_MEM) { 965 chunk->src_addr = dev_addr; 966 chunk->dst_addr = mem_addr; 967 } else { 968 chunk->src_addr = mem_addr; 969 chunk->dst_addr = dev_addr; 970 } 971 972 chunk->size = size; 973 974 dev_dbg(chan->chan.device->dev, 975 "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n", 976 chan->index, chunk, desc, i, sg, size, len, 977 &chunk->src_addr, &chunk->dst_addr); 978 979 mem_addr += size; 980 if (dir == DMA_MEM_TO_MEM) 981 dev_addr += size; 982 983 len -= size; 984 985 list_add_tail(&chunk->node, &desc->chunks); 986 nchunks++; 987 } 988 } 989 990 desc->nchunks = nchunks; 991 desc->size = full_size; 992 993 /* 994 * Use hardware descriptor lists if possible when more than one chunk 995 * needs to be transferred (otherwise they don't make much sense). 996 * 997 * Source/Destination address should be located in same 4GiB region 998 * in the 40bit address space when it uses Hardware descriptor, 999 * and cross_boundary is checking it. 1000 */ 1001 desc->hwdescs.use = !cross_boundary && nchunks > 1; 1002 if (desc->hwdescs.use) { 1003 if (rcar_dmac_fill_hwdesc(chan, desc) < 0) 1004 desc->hwdescs.use = false; 1005 } 1006 1007 return &desc->async_tx; 1008 } 1009 1010 /* ----------------------------------------------------------------------------- 1011 * DMA engine operations 1012 */ 1013 1014 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan) 1015 { 1016 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1017 int ret; 1018 1019 INIT_LIST_HEAD(&rchan->desc.chunks_free); 1020 INIT_LIST_HEAD(&rchan->desc.pages); 1021 1022 /* Preallocate descriptors. */ 1023 ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL); 1024 if (ret < 0) 1025 return -ENOMEM; 1026 1027 ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL); 1028 if (ret < 0) 1029 return -ENOMEM; 1030 1031 return pm_runtime_get_sync(chan->device->dev); 1032 } 1033 1034 static void rcar_dmac_free_chan_resources(struct dma_chan *chan) 1035 { 1036 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1037 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1038 struct rcar_dmac_chan_map *map = &rchan->map; 1039 struct rcar_dmac_desc_page *page, *_page; 1040 struct rcar_dmac_desc *desc; 1041 LIST_HEAD(list); 1042 1043 /* Protect against ISR */ 1044 spin_lock_irq(&rchan->lock); 1045 rcar_dmac_chan_halt(rchan); 1046 spin_unlock_irq(&rchan->lock); 1047 1048 /* 1049 * Now no new interrupts will occur, but one might already be 1050 * running. Wait for it to finish before freeing resources. 1051 */ 1052 synchronize_irq(rchan->irq); 1053 1054 if (rchan->mid_rid >= 0) { 1055 /* The caller is holding dma_list_mutex */ 1056 clear_bit(rchan->mid_rid, dmac->modules); 1057 rchan->mid_rid = -EINVAL; 1058 } 1059 1060 list_splice_init(&rchan->desc.free, &list); 1061 list_splice_init(&rchan->desc.pending, &list); 1062 list_splice_init(&rchan->desc.active, &list); 1063 list_splice_init(&rchan->desc.done, &list); 1064 list_splice_init(&rchan->desc.wait, &list); 1065 1066 rchan->desc.running = NULL; 1067 1068 list_for_each_entry(desc, &list, node) 1069 rcar_dmac_realloc_hwdesc(rchan, desc, 0); 1070 1071 list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) { 1072 list_del(&page->node); 1073 free_page((unsigned long)page); 1074 } 1075 1076 /* Remove slave mapping if present. */ 1077 if (map->slave.xfer_size) { 1078 dma_unmap_resource(chan->device->dev, map->addr, 1079 map->slave.xfer_size, map->dir, 0); 1080 map->slave.xfer_size = 0; 1081 } 1082 1083 pm_runtime_put(chan->device->dev); 1084 } 1085 1086 static struct dma_async_tx_descriptor * 1087 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 1088 dma_addr_t dma_src, size_t len, unsigned long flags) 1089 { 1090 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1091 struct scatterlist sgl; 1092 1093 if (!len) 1094 return NULL; 1095 1096 sg_init_table(&sgl, 1); 1097 sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len, 1098 offset_in_page(dma_src)); 1099 sg_dma_address(&sgl) = dma_src; 1100 sg_dma_len(&sgl) = len; 1101 1102 return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest, 1103 DMA_MEM_TO_MEM, flags, false); 1104 } 1105 1106 static int rcar_dmac_map_slave_addr(struct dma_chan *chan, 1107 enum dma_transfer_direction dir) 1108 { 1109 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1110 struct rcar_dmac_chan_map *map = &rchan->map; 1111 phys_addr_t dev_addr; 1112 size_t dev_size; 1113 enum dma_data_direction dev_dir; 1114 1115 if (dir == DMA_DEV_TO_MEM) { 1116 dev_addr = rchan->src.slave_addr; 1117 dev_size = rchan->src.xfer_size; 1118 dev_dir = DMA_TO_DEVICE; 1119 } else { 1120 dev_addr = rchan->dst.slave_addr; 1121 dev_size = rchan->dst.xfer_size; 1122 dev_dir = DMA_FROM_DEVICE; 1123 } 1124 1125 /* Reuse current map if possible. */ 1126 if (dev_addr == map->slave.slave_addr && 1127 dev_size == map->slave.xfer_size && 1128 dev_dir == map->dir) 1129 return 0; 1130 1131 /* Remove old mapping if present. */ 1132 if (map->slave.xfer_size) 1133 dma_unmap_resource(chan->device->dev, map->addr, 1134 map->slave.xfer_size, map->dir, 0); 1135 map->slave.xfer_size = 0; 1136 1137 /* Create new slave address map. */ 1138 map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size, 1139 dev_dir, 0); 1140 1141 if (dma_mapping_error(chan->device->dev, map->addr)) { 1142 dev_err(chan->device->dev, 1143 "chan%u: failed to map %zx@%pap", rchan->index, 1144 dev_size, &dev_addr); 1145 return -EIO; 1146 } 1147 1148 dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n", 1149 rchan->index, dev_size, &dev_addr, &map->addr, 1150 dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE"); 1151 1152 map->slave.slave_addr = dev_addr; 1153 map->slave.xfer_size = dev_size; 1154 map->dir = dev_dir; 1155 1156 return 0; 1157 } 1158 1159 static struct dma_async_tx_descriptor * 1160 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 1161 unsigned int sg_len, enum dma_transfer_direction dir, 1162 unsigned long flags, void *context) 1163 { 1164 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1165 1166 /* Someone calling slave DMA on a generic channel? */ 1167 if (rchan->mid_rid < 0 || !sg_len) { 1168 dev_warn(chan->device->dev, 1169 "%s: bad parameter: len=%d, id=%d\n", 1170 __func__, sg_len, rchan->mid_rid); 1171 return NULL; 1172 } 1173 1174 if (rcar_dmac_map_slave_addr(chan, dir)) 1175 return NULL; 1176 1177 return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1178 dir, flags, false); 1179 } 1180 1181 #define RCAR_DMAC_MAX_SG_LEN 32 1182 1183 static struct dma_async_tx_descriptor * 1184 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, 1185 size_t buf_len, size_t period_len, 1186 enum dma_transfer_direction dir, unsigned long flags) 1187 { 1188 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1189 struct dma_async_tx_descriptor *desc; 1190 struct scatterlist *sgl; 1191 unsigned int sg_len; 1192 unsigned int i; 1193 1194 /* Someone calling slave DMA on a generic channel? */ 1195 if (rchan->mid_rid < 0 || buf_len < period_len) { 1196 dev_warn(chan->device->dev, 1197 "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n", 1198 __func__, buf_len, period_len, rchan->mid_rid); 1199 return NULL; 1200 } 1201 1202 if (rcar_dmac_map_slave_addr(chan, dir)) 1203 return NULL; 1204 1205 sg_len = buf_len / period_len; 1206 if (sg_len > RCAR_DMAC_MAX_SG_LEN) { 1207 dev_err(chan->device->dev, 1208 "chan%u: sg length %d exceds limit %d", 1209 rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); 1210 return NULL; 1211 } 1212 1213 /* 1214 * Allocate the sg list dynamically as it would consume too much stack 1215 * space. 1216 */ 1217 sgl = kcalloc(sg_len, sizeof(*sgl), GFP_NOWAIT); 1218 if (!sgl) 1219 return NULL; 1220 1221 sg_init_table(sgl, sg_len); 1222 1223 for (i = 0; i < sg_len; ++i) { 1224 dma_addr_t src = buf_addr + (period_len * i); 1225 1226 sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len, 1227 offset_in_page(src)); 1228 sg_dma_address(&sgl[i]) = src; 1229 sg_dma_len(&sgl[i]) = period_len; 1230 } 1231 1232 desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr, 1233 dir, flags, true); 1234 1235 kfree(sgl); 1236 return desc; 1237 } 1238 1239 static int rcar_dmac_device_config(struct dma_chan *chan, 1240 struct dma_slave_config *cfg) 1241 { 1242 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1243 1244 /* 1245 * We could lock this, but you shouldn't be configuring the 1246 * channel, while using it... 1247 */ 1248 rchan->src.slave_addr = cfg->src_addr; 1249 rchan->dst.slave_addr = cfg->dst_addr; 1250 rchan->src.xfer_size = cfg->src_addr_width; 1251 rchan->dst.xfer_size = cfg->dst_addr_width; 1252 1253 return 0; 1254 } 1255 1256 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan) 1257 { 1258 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1259 unsigned long flags; 1260 1261 spin_lock_irqsave(&rchan->lock, flags); 1262 rcar_dmac_chan_halt(rchan); 1263 spin_unlock_irqrestore(&rchan->lock, flags); 1264 1265 /* 1266 * FIXME: No new interrupt can occur now, but the IRQ thread might still 1267 * be running. 1268 */ 1269 1270 rcar_dmac_chan_reinit(rchan); 1271 1272 return 0; 1273 } 1274 1275 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, 1276 dma_cookie_t cookie) 1277 { 1278 struct rcar_dmac_desc *desc = chan->desc.running; 1279 struct rcar_dmac_xfer_chunk *running = NULL; 1280 struct rcar_dmac_xfer_chunk *chunk; 1281 enum dma_status status; 1282 unsigned int residue = 0; 1283 unsigned int dptr = 0; 1284 1285 if (!desc) 1286 return 0; 1287 1288 /* 1289 * If the cookie corresponds to a descriptor that has been completed 1290 * there is no residue. The same check has already been performed by the 1291 * caller but without holding the channel lock, so the descriptor could 1292 * now be complete. 1293 */ 1294 status = dma_cookie_status(&chan->chan, cookie, NULL); 1295 if (status == DMA_COMPLETE) 1296 return 0; 1297 1298 /* 1299 * If the cookie doesn't correspond to the currently running transfer 1300 * then the descriptor hasn't been processed yet, and the residue is 1301 * equal to the full descriptor size. 1302 * Also, a client driver is possible to call this function before 1303 * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running" 1304 * will be the next descriptor, and the done list will appear. So, if 1305 * the argument cookie matches the done list's cookie, we can assume 1306 * the residue is zero. 1307 */ 1308 if (cookie != desc->async_tx.cookie) { 1309 list_for_each_entry(desc, &chan->desc.done, node) { 1310 if (cookie == desc->async_tx.cookie) 1311 return 0; 1312 } 1313 list_for_each_entry(desc, &chan->desc.pending, node) { 1314 if (cookie == desc->async_tx.cookie) 1315 return desc->size; 1316 } 1317 list_for_each_entry(desc, &chan->desc.active, node) { 1318 if (cookie == desc->async_tx.cookie) 1319 return desc->size; 1320 } 1321 1322 /* 1323 * No descriptor found for the cookie, there's thus no residue. 1324 * This shouldn't happen if the calling driver passes a correct 1325 * cookie value. 1326 */ 1327 WARN(1, "No descriptor for cookie!"); 1328 return 0; 1329 } 1330 1331 /* 1332 * In descriptor mode the descriptor running pointer is not maintained 1333 * by the interrupt handler, find the running descriptor from the 1334 * descriptor pointer field in the CHCRB register. In non-descriptor 1335 * mode just use the running descriptor pointer. 1336 */ 1337 if (desc->hwdescs.use) { 1338 dptr = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1339 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1340 if (dptr == 0) 1341 dptr = desc->nchunks; 1342 dptr--; 1343 WARN_ON(dptr >= desc->nchunks); 1344 } else { 1345 running = desc->running; 1346 } 1347 1348 /* Compute the size of all chunks still to be transferred. */ 1349 list_for_each_entry_reverse(chunk, &desc->chunks, node) { 1350 if (chunk == running || ++dptr == desc->nchunks) 1351 break; 1352 1353 residue += chunk->size; 1354 } 1355 1356 /* Add the residue for the current chunk. */ 1357 residue += rcar_dmac_chan_read(chan, RCAR_DMATCRB) << desc->xfer_shift; 1358 1359 return residue; 1360 } 1361 1362 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, 1363 dma_cookie_t cookie, 1364 struct dma_tx_state *txstate) 1365 { 1366 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1367 enum dma_status status; 1368 unsigned long flags; 1369 unsigned int residue; 1370 1371 status = dma_cookie_status(chan, cookie, txstate); 1372 if (status == DMA_COMPLETE || !txstate) 1373 return status; 1374 1375 spin_lock_irqsave(&rchan->lock, flags); 1376 residue = rcar_dmac_chan_get_residue(rchan, cookie); 1377 spin_unlock_irqrestore(&rchan->lock, flags); 1378 1379 /* if there's no residue, the cookie is complete */ 1380 if (!residue) 1381 return DMA_COMPLETE; 1382 1383 dma_set_residue(txstate, residue); 1384 1385 return status; 1386 } 1387 1388 static void rcar_dmac_issue_pending(struct dma_chan *chan) 1389 { 1390 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1391 unsigned long flags; 1392 1393 spin_lock_irqsave(&rchan->lock, flags); 1394 1395 if (list_empty(&rchan->desc.pending)) 1396 goto done; 1397 1398 /* Append the pending list to the active list. */ 1399 list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active); 1400 1401 /* 1402 * If no transfer is running pick the first descriptor from the active 1403 * list and start the transfer. 1404 */ 1405 if (!rchan->desc.running) { 1406 struct rcar_dmac_desc *desc; 1407 1408 desc = list_first_entry(&rchan->desc.active, 1409 struct rcar_dmac_desc, node); 1410 rchan->desc.running = desc; 1411 1412 rcar_dmac_chan_start_xfer(rchan); 1413 } 1414 1415 done: 1416 spin_unlock_irqrestore(&rchan->lock, flags); 1417 } 1418 1419 static void rcar_dmac_device_synchronize(struct dma_chan *chan) 1420 { 1421 struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan); 1422 1423 synchronize_irq(rchan->irq); 1424 } 1425 1426 /* ----------------------------------------------------------------------------- 1427 * IRQ handling 1428 */ 1429 1430 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan) 1431 { 1432 struct rcar_dmac_desc *desc = chan->desc.running; 1433 unsigned int stage; 1434 1435 if (WARN_ON(!desc || !desc->cyclic)) { 1436 /* 1437 * This should never happen, there should always be a running 1438 * cyclic descriptor when a descriptor stage end interrupt is 1439 * triggered. Warn and return. 1440 */ 1441 return IRQ_NONE; 1442 } 1443 1444 /* Program the interrupt pointer to the next stage. */ 1445 stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) & 1446 RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT; 1447 rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage)); 1448 1449 return IRQ_WAKE_THREAD; 1450 } 1451 1452 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan) 1453 { 1454 struct rcar_dmac_desc *desc = chan->desc.running; 1455 irqreturn_t ret = IRQ_WAKE_THREAD; 1456 1457 if (WARN_ON_ONCE(!desc)) { 1458 /* 1459 * This should never happen, there should always be a running 1460 * descriptor when a transfer end interrupt is triggered. Warn 1461 * and return. 1462 */ 1463 return IRQ_NONE; 1464 } 1465 1466 /* 1467 * The transfer end interrupt isn't generated for each chunk when using 1468 * descriptor mode. Only update the running chunk pointer in 1469 * non-descriptor mode. 1470 */ 1471 if (!desc->hwdescs.use) { 1472 /* 1473 * If we haven't completed the last transfer chunk simply move 1474 * to the next one. Only wake the IRQ thread if the transfer is 1475 * cyclic. 1476 */ 1477 if (!list_is_last(&desc->running->node, &desc->chunks)) { 1478 desc->running = list_next_entry(desc->running, node); 1479 if (!desc->cyclic) 1480 ret = IRQ_HANDLED; 1481 goto done; 1482 } 1483 1484 /* 1485 * We've completed the last transfer chunk. If the transfer is 1486 * cyclic, move back to the first one. 1487 */ 1488 if (desc->cyclic) { 1489 desc->running = 1490 list_first_entry(&desc->chunks, 1491 struct rcar_dmac_xfer_chunk, 1492 node); 1493 goto done; 1494 } 1495 } 1496 1497 /* The descriptor is complete, move it to the done list. */ 1498 list_move_tail(&desc->node, &chan->desc.done); 1499 1500 /* Queue the next descriptor, if any. */ 1501 if (!list_empty(&chan->desc.active)) 1502 chan->desc.running = list_first_entry(&chan->desc.active, 1503 struct rcar_dmac_desc, 1504 node); 1505 else 1506 chan->desc.running = NULL; 1507 1508 done: 1509 if (chan->desc.running) 1510 rcar_dmac_chan_start_xfer(chan); 1511 1512 return ret; 1513 } 1514 1515 static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) 1516 { 1517 u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE; 1518 struct rcar_dmac_chan *chan = dev; 1519 irqreturn_t ret = IRQ_NONE; 1520 bool reinit = false; 1521 u32 chcr; 1522 1523 spin_lock(&chan->lock); 1524 1525 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); 1526 if (chcr & RCAR_DMACHCR_CAE) { 1527 struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device); 1528 1529 /* 1530 * We don't need to call rcar_dmac_chan_halt() 1531 * because channel is already stopped in error case. 1532 * We need to clear register and check DE bit as recovery. 1533 */ 1534 rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index); 1535 rcar_dmac_chcr_de_barrier(chan); 1536 reinit = true; 1537 goto spin_lock_end; 1538 } 1539 1540 if (chcr & RCAR_DMACHCR_TE) 1541 mask |= RCAR_DMACHCR_DE; 1542 rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask); 1543 if (mask & RCAR_DMACHCR_DE) 1544 rcar_dmac_chcr_de_barrier(chan); 1545 1546 if (chcr & RCAR_DMACHCR_DSE) 1547 ret |= rcar_dmac_isr_desc_stage_end(chan); 1548 1549 if (chcr & RCAR_DMACHCR_TE) 1550 ret |= rcar_dmac_isr_transfer_end(chan); 1551 1552 spin_lock_end: 1553 spin_unlock(&chan->lock); 1554 1555 if (reinit) { 1556 dev_err(chan->chan.device->dev, "Channel Address Error\n"); 1557 1558 rcar_dmac_chan_reinit(chan); 1559 ret = IRQ_HANDLED; 1560 } 1561 1562 return ret; 1563 } 1564 1565 static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev) 1566 { 1567 struct rcar_dmac_chan *chan = dev; 1568 struct rcar_dmac_desc *desc; 1569 struct dmaengine_desc_callback cb; 1570 1571 spin_lock_irq(&chan->lock); 1572 1573 /* For cyclic transfers notify the user after every chunk. */ 1574 if (chan->desc.running && chan->desc.running->cyclic) { 1575 desc = chan->desc.running; 1576 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1577 1578 if (dmaengine_desc_callback_valid(&cb)) { 1579 spin_unlock_irq(&chan->lock); 1580 dmaengine_desc_callback_invoke(&cb, NULL); 1581 spin_lock_irq(&chan->lock); 1582 } 1583 } 1584 1585 /* 1586 * Call the callback function for all descriptors on the done list and 1587 * move them to the ack wait list. 1588 */ 1589 while (!list_empty(&chan->desc.done)) { 1590 desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc, 1591 node); 1592 dma_cookie_complete(&desc->async_tx); 1593 list_del(&desc->node); 1594 1595 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1596 if (dmaengine_desc_callback_valid(&cb)) { 1597 spin_unlock_irq(&chan->lock); 1598 /* 1599 * We own the only reference to this descriptor, we can 1600 * safely dereference it without holding the channel 1601 * lock. 1602 */ 1603 dmaengine_desc_callback_invoke(&cb, NULL); 1604 spin_lock_irq(&chan->lock); 1605 } 1606 1607 list_add_tail(&desc->node, &chan->desc.wait); 1608 } 1609 1610 spin_unlock_irq(&chan->lock); 1611 1612 /* Recycle all acked descriptors. */ 1613 rcar_dmac_desc_recycle_acked(chan); 1614 1615 return IRQ_HANDLED; 1616 } 1617 1618 /* ----------------------------------------------------------------------------- 1619 * OF xlate and channel filter 1620 */ 1621 1622 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg) 1623 { 1624 struct rcar_dmac *dmac = to_rcar_dmac(chan->device); 1625 struct of_phandle_args *dma_spec = arg; 1626 1627 /* 1628 * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate 1629 * function knows from which device it wants to allocate a channel from, 1630 * and would be perfectly capable of selecting the channel it wants. 1631 * Forcing it to call dma_request_channel() and iterate through all 1632 * channels from all controllers is just pointless. 1633 */ 1634 if (chan->device->device_config != rcar_dmac_device_config || 1635 dma_spec->np != chan->device->dev->of_node) 1636 return false; 1637 1638 return !test_and_set_bit(dma_spec->args[0], dmac->modules); 1639 } 1640 1641 static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, 1642 struct of_dma *ofdma) 1643 { 1644 struct rcar_dmac_chan *rchan; 1645 struct dma_chan *chan; 1646 dma_cap_mask_t mask; 1647 1648 if (dma_spec->args_count != 1) 1649 return NULL; 1650 1651 /* Only slave DMA channels can be allocated via DT */ 1652 dma_cap_zero(mask); 1653 dma_cap_set(DMA_SLAVE, mask); 1654 1655 chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec); 1656 if (!chan) 1657 return NULL; 1658 1659 rchan = to_rcar_dmac_chan(chan); 1660 rchan->mid_rid = dma_spec->args[0]; 1661 1662 return chan; 1663 } 1664 1665 /* ----------------------------------------------------------------------------- 1666 * Power management 1667 */ 1668 1669 #ifdef CONFIG_PM 1670 static int rcar_dmac_runtime_suspend(struct device *dev) 1671 { 1672 return 0; 1673 } 1674 1675 static int rcar_dmac_runtime_resume(struct device *dev) 1676 { 1677 struct rcar_dmac *dmac = dev_get_drvdata(dev); 1678 1679 return rcar_dmac_init(dmac); 1680 } 1681 #endif 1682 1683 static const struct dev_pm_ops rcar_dmac_pm = { 1684 /* 1685 * TODO for system sleep/resume: 1686 * - Wait for the current transfer to complete and stop the device, 1687 * - Resume transfers, if any. 1688 */ 1689 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 1690 pm_runtime_force_resume) 1691 SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, 1692 NULL) 1693 }; 1694 1695 /* ----------------------------------------------------------------------------- 1696 * Probe and remove 1697 */ 1698 1699 static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, 1700 struct rcar_dmac_chan *rchan, 1701 unsigned int index) 1702 { 1703 struct platform_device *pdev = to_platform_device(dmac->dev); 1704 struct dma_chan *chan = &rchan->chan; 1705 char pdev_irqname[5]; 1706 char *irqname; 1707 int ret; 1708 1709 rchan->index = index; 1710 rchan->iomem = dmac->iomem + RCAR_DMAC_CHAN_OFFSET(index); 1711 rchan->mid_rid = -EINVAL; 1712 1713 spin_lock_init(&rchan->lock); 1714 1715 INIT_LIST_HEAD(&rchan->desc.free); 1716 INIT_LIST_HEAD(&rchan->desc.pending); 1717 INIT_LIST_HEAD(&rchan->desc.active); 1718 INIT_LIST_HEAD(&rchan->desc.done); 1719 INIT_LIST_HEAD(&rchan->desc.wait); 1720 1721 /* Request the channel interrupt. */ 1722 sprintf(pdev_irqname, "ch%u", index); 1723 rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); 1724 if (rchan->irq < 0) { 1725 dev_err(dmac->dev, "no IRQ specified for channel %u\n", index); 1726 return -ENODEV; 1727 } 1728 1729 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", 1730 dev_name(dmac->dev), index); 1731 if (!irqname) 1732 return -ENOMEM; 1733 1734 /* 1735 * Initialize the DMA engine channel and add it to the DMA engine 1736 * channels list. 1737 */ 1738 chan->device = &dmac->engine; 1739 dma_cookie_init(chan); 1740 1741 list_add_tail(&chan->device_node, &dmac->engine.channels); 1742 1743 ret = devm_request_threaded_irq(dmac->dev, rchan->irq, 1744 rcar_dmac_isr_channel, 1745 rcar_dmac_isr_channel_thread, 0, 1746 irqname, rchan); 1747 if (ret) { 1748 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", 1749 rchan->irq, ret); 1750 return ret; 1751 } 1752 1753 return 0; 1754 } 1755 1756 static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac) 1757 { 1758 struct device_node *np = dev->of_node; 1759 int ret; 1760 1761 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); 1762 if (ret < 0) { 1763 dev_err(dev, "unable to read dma-channels property\n"); 1764 return ret; 1765 } 1766 1767 if (dmac->n_channels <= 0 || dmac->n_channels >= 100) { 1768 dev_err(dev, "invalid number of channels %u\n", 1769 dmac->n_channels); 1770 return -EINVAL; 1771 } 1772 1773 return 0; 1774 } 1775 1776 static int rcar_dmac_probe(struct platform_device *pdev) 1777 { 1778 const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | 1779 DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | 1780 DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | 1781 DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; 1782 unsigned int channels_offset = 0; 1783 struct dma_device *engine; 1784 struct rcar_dmac *dmac; 1785 struct resource *mem; 1786 unsigned int i; 1787 int ret; 1788 1789 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1790 if (!dmac) 1791 return -ENOMEM; 1792 1793 dmac->dev = &pdev->dev; 1794 platform_set_drvdata(pdev, dmac); 1795 dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); 1796 1797 ret = rcar_dmac_parse_of(&pdev->dev, dmac); 1798 if (ret < 0) 1799 return ret; 1800 1801 /* 1802 * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be 1803 * flushed correctly, resulting in memory corruption. DMAC 0 channel 0 1804 * is connected to microTLB 0 on currently supported platforms, so we 1805 * can't use it with the IPMMU. As the IOMMU API operates at the device 1806 * level we can't disable it selectively, so ignore channel 0 for now if 1807 * the device is part of an IOMMU group. 1808 */ 1809 if (pdev->dev.iommu_group) { 1810 dmac->n_channels--; 1811 channels_offset = 1; 1812 } 1813 1814 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, 1815 sizeof(*dmac->channels), GFP_KERNEL); 1816 if (!dmac->channels) 1817 return -ENOMEM; 1818 1819 /* Request resources. */ 1820 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1821 dmac->iomem = devm_ioremap_resource(&pdev->dev, mem); 1822 if (IS_ERR(dmac->iomem)) 1823 return PTR_ERR(dmac->iomem); 1824 1825 /* Enable runtime PM and initialize the device. */ 1826 pm_runtime_enable(&pdev->dev); 1827 ret = pm_runtime_get_sync(&pdev->dev); 1828 if (ret < 0) { 1829 dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); 1830 return ret; 1831 } 1832 1833 ret = rcar_dmac_init(dmac); 1834 pm_runtime_put(&pdev->dev); 1835 1836 if (ret) { 1837 dev_err(&pdev->dev, "failed to reset device\n"); 1838 goto error; 1839 } 1840 1841 /* Initialize engine */ 1842 engine = &dmac->engine; 1843 1844 dma_cap_set(DMA_MEMCPY, engine->cap_mask); 1845 dma_cap_set(DMA_SLAVE, engine->cap_mask); 1846 1847 engine->dev = &pdev->dev; 1848 engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE); 1849 1850 engine->src_addr_widths = widths; 1851 engine->dst_addr_widths = widths; 1852 engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1853 engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1854 1855 engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources; 1856 engine->device_free_chan_resources = rcar_dmac_free_chan_resources; 1857 engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy; 1858 engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg; 1859 engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic; 1860 engine->device_config = rcar_dmac_device_config; 1861 engine->device_pause = rcar_dmac_chan_pause; 1862 engine->device_terminate_all = rcar_dmac_chan_terminate_all; 1863 engine->device_tx_status = rcar_dmac_tx_status; 1864 engine->device_issue_pending = rcar_dmac_issue_pending; 1865 engine->device_synchronize = rcar_dmac_device_synchronize; 1866 1867 INIT_LIST_HEAD(&engine->channels); 1868 1869 for (i = 0; i < dmac->n_channels; ++i) { 1870 ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], 1871 i + channels_offset); 1872 if (ret < 0) 1873 goto error; 1874 } 1875 1876 /* Register the DMAC as a DMA provider for DT. */ 1877 ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate, 1878 NULL); 1879 if (ret < 0) 1880 goto error; 1881 1882 /* 1883 * Register the DMA engine device. 1884 * 1885 * Default transfer size of 32 bytes requires 32-byte alignment. 1886 */ 1887 ret = dma_async_device_register(engine); 1888 if (ret < 0) 1889 goto error; 1890 1891 return 0; 1892 1893 error: 1894 of_dma_controller_free(pdev->dev.of_node); 1895 pm_runtime_disable(&pdev->dev); 1896 return ret; 1897 } 1898 1899 static int rcar_dmac_remove(struct platform_device *pdev) 1900 { 1901 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1902 1903 of_dma_controller_free(pdev->dev.of_node); 1904 dma_async_device_unregister(&dmac->engine); 1905 1906 pm_runtime_disable(&pdev->dev); 1907 1908 return 0; 1909 } 1910 1911 static void rcar_dmac_shutdown(struct platform_device *pdev) 1912 { 1913 struct rcar_dmac *dmac = platform_get_drvdata(pdev); 1914 1915 rcar_dmac_stop_all_chan(dmac); 1916 } 1917 1918 static const struct of_device_id rcar_dmac_of_ids[] = { 1919 { .compatible = "renesas,rcar-dmac", }, 1920 { /* Sentinel */ } 1921 }; 1922 MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); 1923 1924 static struct platform_driver rcar_dmac_driver = { 1925 .driver = { 1926 .pm = &rcar_dmac_pm, 1927 .name = "rcar-dmac", 1928 .of_match_table = rcar_dmac_of_ids, 1929 }, 1930 .probe = rcar_dmac_probe, 1931 .remove = rcar_dmac_remove, 1932 .shutdown = rcar_dmac_shutdown, 1933 }; 1934 1935 module_platform_driver(rcar_dmac_driver); 1936 1937 MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver"); 1938 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>"); 1939 MODULE_LICENSE("GPL v2"); 1940