1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Core driver for the Synopsys DesignWare DMA Controller 4 * 5 * Copyright (C) 2007-2008 Atmel Corporation 6 * Copyright (C) 2010-2011 ST Microelectronics 7 * Copyright (C) 2013 Intel Corporation 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/delay.h> 12 #include <linux/dmaengine.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmapool.h> 15 #include <linux/err.h> 16 #include <linux/init.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/log2.h> 20 #include <linux/mm.h> 21 #include <linux/module.h> 22 #include <linux/slab.h> 23 #include <linux/pm_runtime.h> 24 25 #include "../dmaengine.h" 26 #include "internal.h" 27 28 /* 29 * This supports the Synopsys "DesignWare AHB Central DMA Controller", 30 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all 31 * of which use ARM any more). See the "Databook" from Synopsys for 32 * information beyond what licensees probably provide. 33 */ 34 35 /* The set of bus widths supported by the DMA controller */ 36 #define DW_DMA_BUSWIDTHS \ 37 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 38 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 39 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 40 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 41 42 /*----------------------------------------------------------------------*/ 43 44 static struct device *chan2dev(struct dma_chan *chan) 45 { 46 return &chan->dev->device; 47 } 48 49 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc) 50 { 51 return to_dw_desc(dwc->active_list.next); 52 } 53 54 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx) 55 { 56 struct dw_desc *desc = txd_to_dw_desc(tx); 57 struct dw_dma_chan *dwc = to_dw_dma_chan(tx->chan); 58 dma_cookie_t cookie; 59 unsigned long flags; 60 61 spin_lock_irqsave(&dwc->lock, flags); 62 cookie = dma_cookie_assign(tx); 63 64 /* 65 * REVISIT: We should attempt to chain as many descriptors as 66 * possible, perhaps even appending to those already submitted 67 * for DMA. But this is hard to do in a race-free manner. 68 */ 69 70 list_add_tail(&desc->desc_node, &dwc->queue); 71 spin_unlock_irqrestore(&dwc->lock, flags); 72 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", 73 __func__, desc->txd.cookie); 74 75 return cookie; 76 } 77 78 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc) 79 { 80 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 81 struct dw_desc *desc; 82 dma_addr_t phys; 83 84 desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys); 85 if (!desc) 86 return NULL; 87 88 dwc->descs_allocated++; 89 INIT_LIST_HEAD(&desc->tx_list); 90 dma_async_tx_descriptor_init(&desc->txd, &dwc->chan); 91 desc->txd.tx_submit = dwc_tx_submit; 92 desc->txd.flags = DMA_CTRL_ACK; 93 desc->txd.phys = phys; 94 return desc; 95 } 96 97 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc) 98 { 99 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 100 struct dw_desc *child, *_next; 101 102 if (unlikely(!desc)) 103 return; 104 105 list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) { 106 list_del(&child->desc_node); 107 dma_pool_free(dw->desc_pool, child, child->txd.phys); 108 dwc->descs_allocated--; 109 } 110 111 dma_pool_free(dw->desc_pool, desc, desc->txd.phys); 112 dwc->descs_allocated--; 113 } 114 115 static void dwc_initialize(struct dw_dma_chan *dwc) 116 { 117 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 118 119 dw->initialize_chan(dwc); 120 121 /* Enable interrupts */ 122 channel_set_bit(dw, MASK.XFER, dwc->mask); 123 channel_set_bit(dw, MASK.ERROR, dwc->mask); 124 } 125 126 /*----------------------------------------------------------------------*/ 127 128 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc) 129 { 130 dev_err(chan2dev(&dwc->chan), 131 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n", 132 channel_readl(dwc, SAR), 133 channel_readl(dwc, DAR), 134 channel_readl(dwc, LLP), 135 channel_readl(dwc, CTL_HI), 136 channel_readl(dwc, CTL_LO)); 137 } 138 139 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc) 140 { 141 channel_clear_bit(dw, CH_EN, dwc->mask); 142 while (dma_readl(dw, CH_EN) & dwc->mask) 143 cpu_relax(); 144 } 145 146 /*----------------------------------------------------------------------*/ 147 148 /* Perform single block transfer */ 149 static inline void dwc_do_single_block(struct dw_dma_chan *dwc, 150 struct dw_desc *desc) 151 { 152 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 153 u32 ctllo; 154 155 /* 156 * Software emulation of LLP mode relies on interrupts to continue 157 * multi block transfer. 158 */ 159 ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN; 160 161 channel_writel(dwc, SAR, lli_read(desc, sar)); 162 channel_writel(dwc, DAR, lli_read(desc, dar)); 163 channel_writel(dwc, CTL_LO, ctllo); 164 channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi)); 165 channel_set_bit(dw, CH_EN, dwc->mask); 166 167 /* Move pointer to next descriptor */ 168 dwc->tx_node_active = dwc->tx_node_active->next; 169 } 170 171 /* Called with dwc->lock held and bh disabled */ 172 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first) 173 { 174 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 175 u8 lms = DWC_LLP_LMS(dwc->dws.m_master); 176 unsigned long was_soft_llp; 177 178 /* ASSERT: channel is idle */ 179 if (dma_readl(dw, CH_EN) & dwc->mask) { 180 dev_err(chan2dev(&dwc->chan), 181 "%s: BUG: Attempted to start non-idle channel\n", 182 __func__); 183 dwc_dump_chan_regs(dwc); 184 185 /* The tasklet will hopefully advance the queue... */ 186 return; 187 } 188 189 if (dwc->nollp) { 190 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP, 191 &dwc->flags); 192 if (was_soft_llp) { 193 dev_err(chan2dev(&dwc->chan), 194 "BUG: Attempted to start new LLP transfer inside ongoing one\n"); 195 return; 196 } 197 198 dwc_initialize(dwc); 199 200 first->residue = first->total_len; 201 dwc->tx_node_active = &first->tx_list; 202 203 /* Submit first block */ 204 dwc_do_single_block(dwc, first); 205 206 return; 207 } 208 209 dwc_initialize(dwc); 210 211 channel_writel(dwc, LLP, first->txd.phys | lms); 212 channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN); 213 channel_writel(dwc, CTL_HI, 0); 214 channel_set_bit(dw, CH_EN, dwc->mask); 215 } 216 217 static void dwc_dostart_first_queued(struct dw_dma_chan *dwc) 218 { 219 struct dw_desc *desc; 220 221 if (list_empty(&dwc->queue)) 222 return; 223 224 list_move(dwc->queue.next, &dwc->active_list); 225 desc = dwc_first_active(dwc); 226 dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie); 227 dwc_dostart(dwc, desc); 228 } 229 230 /*----------------------------------------------------------------------*/ 231 232 static void 233 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc, 234 bool callback_required) 235 { 236 struct dma_async_tx_descriptor *txd = &desc->txd; 237 struct dw_desc *child; 238 unsigned long flags; 239 struct dmaengine_desc_callback cb; 240 241 dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie); 242 243 spin_lock_irqsave(&dwc->lock, flags); 244 dma_cookie_complete(txd); 245 if (callback_required) 246 dmaengine_desc_get_callback(txd, &cb); 247 else 248 memset(&cb, 0, sizeof(cb)); 249 250 /* async_tx_ack */ 251 list_for_each_entry(child, &desc->tx_list, desc_node) 252 async_tx_ack(&child->txd); 253 async_tx_ack(&desc->txd); 254 dwc_desc_put(dwc, desc); 255 spin_unlock_irqrestore(&dwc->lock, flags); 256 257 dmaengine_desc_callback_invoke(&cb, NULL); 258 } 259 260 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc) 261 { 262 struct dw_desc *desc, *_desc; 263 LIST_HEAD(list); 264 unsigned long flags; 265 266 spin_lock_irqsave(&dwc->lock, flags); 267 if (dma_readl(dw, CH_EN) & dwc->mask) { 268 dev_err(chan2dev(&dwc->chan), 269 "BUG: XFER bit set, but channel not idle!\n"); 270 271 /* Try to continue after resetting the channel... */ 272 dwc_chan_disable(dw, dwc); 273 } 274 275 /* 276 * Submit queued descriptors ASAP, i.e. before we go through 277 * the completed ones. 278 */ 279 list_splice_init(&dwc->active_list, &list); 280 dwc_dostart_first_queued(dwc); 281 282 spin_unlock_irqrestore(&dwc->lock, flags); 283 284 list_for_each_entry_safe(desc, _desc, &list, desc_node) 285 dwc_descriptor_complete(dwc, desc, true); 286 } 287 288 /* Returns how many bytes were already received from source */ 289 static inline u32 dwc_get_sent(struct dw_dma_chan *dwc) 290 { 291 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 292 u32 ctlhi = channel_readl(dwc, CTL_HI); 293 u32 ctllo = channel_readl(dwc, CTL_LO); 294 295 return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7); 296 } 297 298 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) 299 { 300 dma_addr_t llp; 301 struct dw_desc *desc, *_desc; 302 struct dw_desc *child; 303 u32 status_xfer; 304 unsigned long flags; 305 306 spin_lock_irqsave(&dwc->lock, flags); 307 llp = channel_readl(dwc, LLP); 308 status_xfer = dma_readl(dw, RAW.XFER); 309 310 if (status_xfer & dwc->mask) { 311 /* Everything we've submitted is done */ 312 dma_writel(dw, CLEAR.XFER, dwc->mask); 313 314 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { 315 struct list_head *head, *active = dwc->tx_node_active; 316 317 /* 318 * We are inside first active descriptor. 319 * Otherwise something is really wrong. 320 */ 321 desc = dwc_first_active(dwc); 322 323 head = &desc->tx_list; 324 if (active != head) { 325 /* Update residue to reflect last sent descriptor */ 326 if (active == head->next) 327 desc->residue -= desc->len; 328 else 329 desc->residue -= to_dw_desc(active->prev)->len; 330 331 child = to_dw_desc(active); 332 333 /* Submit next block */ 334 dwc_do_single_block(dwc, child); 335 336 spin_unlock_irqrestore(&dwc->lock, flags); 337 return; 338 } 339 340 /* We are done here */ 341 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); 342 } 343 344 spin_unlock_irqrestore(&dwc->lock, flags); 345 346 dwc_complete_all(dw, dwc); 347 return; 348 } 349 350 if (list_empty(&dwc->active_list)) { 351 spin_unlock_irqrestore(&dwc->lock, flags); 352 return; 353 } 354 355 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) { 356 dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__); 357 spin_unlock_irqrestore(&dwc->lock, flags); 358 return; 359 } 360 361 dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp); 362 363 list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) { 364 /* Initial residue value */ 365 desc->residue = desc->total_len; 366 367 /* Check first descriptors addr */ 368 if (desc->txd.phys == DWC_LLP_LOC(llp)) { 369 spin_unlock_irqrestore(&dwc->lock, flags); 370 return; 371 } 372 373 /* Check first descriptors llp */ 374 if (lli_read(desc, llp) == llp) { 375 /* This one is currently in progress */ 376 desc->residue -= dwc_get_sent(dwc); 377 spin_unlock_irqrestore(&dwc->lock, flags); 378 return; 379 } 380 381 desc->residue -= desc->len; 382 list_for_each_entry(child, &desc->tx_list, desc_node) { 383 if (lli_read(child, llp) == llp) { 384 /* Currently in progress */ 385 desc->residue -= dwc_get_sent(dwc); 386 spin_unlock_irqrestore(&dwc->lock, flags); 387 return; 388 } 389 desc->residue -= child->len; 390 } 391 392 /* 393 * No descriptors so far seem to be in progress, i.e. 394 * this one must be done. 395 */ 396 spin_unlock_irqrestore(&dwc->lock, flags); 397 dwc_descriptor_complete(dwc, desc, true); 398 spin_lock_irqsave(&dwc->lock, flags); 399 } 400 401 dev_err(chan2dev(&dwc->chan), 402 "BUG: All descriptors done, but channel not idle!\n"); 403 404 /* Try to continue after resetting the channel... */ 405 dwc_chan_disable(dw, dwc); 406 407 dwc_dostart_first_queued(dwc); 408 spin_unlock_irqrestore(&dwc->lock, flags); 409 } 410 411 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc) 412 { 413 dev_crit(chan2dev(&dwc->chan), " desc: s0x%x d0x%x l0x%x c0x%x:%x\n", 414 lli_read(desc, sar), 415 lli_read(desc, dar), 416 lli_read(desc, llp), 417 lli_read(desc, ctlhi), 418 lli_read(desc, ctllo)); 419 } 420 421 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc) 422 { 423 struct dw_desc *bad_desc; 424 struct dw_desc *child; 425 unsigned long flags; 426 427 dwc_scan_descriptors(dw, dwc); 428 429 spin_lock_irqsave(&dwc->lock, flags); 430 431 /* 432 * The descriptor currently at the head of the active list is 433 * borked. Since we don't have any way to report errors, we'll 434 * just have to scream loudly and try to carry on. 435 */ 436 bad_desc = dwc_first_active(dwc); 437 list_del_init(&bad_desc->desc_node); 438 list_move(dwc->queue.next, dwc->active_list.prev); 439 440 /* Clear the error flag and try to restart the controller */ 441 dma_writel(dw, CLEAR.ERROR, dwc->mask); 442 if (!list_empty(&dwc->active_list)) 443 dwc_dostart(dwc, dwc_first_active(dwc)); 444 445 /* 446 * WARN may seem harsh, but since this only happens 447 * when someone submits a bad physical address in a 448 * descriptor, we should consider ourselves lucky that the 449 * controller flagged an error instead of scribbling over 450 * random memory locations. 451 */ 452 dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n" 453 " cookie: %d\n", bad_desc->txd.cookie); 454 dwc_dump_lli(dwc, bad_desc); 455 list_for_each_entry(child, &bad_desc->tx_list, desc_node) 456 dwc_dump_lli(dwc, child); 457 458 spin_unlock_irqrestore(&dwc->lock, flags); 459 460 /* Pretend the descriptor completed successfully */ 461 dwc_descriptor_complete(dwc, bad_desc, true); 462 } 463 464 static void dw_dma_tasklet(struct tasklet_struct *t) 465 { 466 struct dw_dma *dw = from_tasklet(dw, t, tasklet); 467 struct dw_dma_chan *dwc; 468 u32 status_xfer; 469 u32 status_err; 470 unsigned int i; 471 472 status_xfer = dma_readl(dw, RAW.XFER); 473 status_err = dma_readl(dw, RAW.ERROR); 474 475 dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err); 476 477 for (i = 0; i < dw->dma.chancnt; i++) { 478 dwc = &dw->chan[i]; 479 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) 480 dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n"); 481 else if (status_err & (1 << i)) 482 dwc_handle_error(dw, dwc); 483 else if (status_xfer & (1 << i)) 484 dwc_scan_descriptors(dw, dwc); 485 } 486 487 /* Re-enable interrupts */ 488 channel_set_bit(dw, MASK.XFER, dw->all_chan_mask); 489 channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask); 490 } 491 492 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id) 493 { 494 struct dw_dma *dw = dev_id; 495 u32 status; 496 497 /* Check if we have any interrupt from the DMAC which is not in use */ 498 if (!dw->in_use) 499 return IRQ_NONE; 500 501 status = dma_readl(dw, STATUS_INT); 502 dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status); 503 504 /* Check if we have any interrupt from the DMAC */ 505 if (!status) 506 return IRQ_NONE; 507 508 /* 509 * Just disable the interrupts. We'll turn them back on in the 510 * softirq handler. 511 */ 512 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask); 513 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask); 514 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask); 515 516 status = dma_readl(dw, STATUS_INT); 517 if (status) { 518 dev_err(dw->dma.dev, 519 "BUG: Unexpected interrupts pending: 0x%x\n", 520 status); 521 522 /* Try to recover */ 523 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1); 524 channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1); 525 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1); 526 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1); 527 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1); 528 } 529 530 tasklet_schedule(&dw->tasklet); 531 532 return IRQ_HANDLED; 533 } 534 535 /*----------------------------------------------------------------------*/ 536 537 static struct dma_async_tx_descriptor * 538 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 539 size_t len, unsigned long flags) 540 { 541 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 542 struct dw_dma *dw = to_dw_dma(chan->device); 543 struct dw_desc *desc; 544 struct dw_desc *first; 545 struct dw_desc *prev; 546 size_t xfer_count; 547 size_t offset; 548 u8 m_master = dwc->dws.m_master; 549 unsigned int src_width; 550 unsigned int dst_width; 551 unsigned int data_width = dw->pdata->data_width[m_master]; 552 u32 ctllo, ctlhi; 553 u8 lms = DWC_LLP_LMS(m_master); 554 555 dev_vdbg(chan2dev(chan), 556 "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__, 557 &dest, &src, len, flags); 558 559 if (unlikely(!len)) { 560 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); 561 return NULL; 562 } 563 564 dwc->direction = DMA_MEM_TO_MEM; 565 566 src_width = dst_width = __ffs(data_width | src | dest | len); 567 568 ctllo = dw->prepare_ctllo(dwc) 569 | DWC_CTLL_DST_WIDTH(dst_width) 570 | DWC_CTLL_SRC_WIDTH(src_width) 571 | DWC_CTLL_DST_INC 572 | DWC_CTLL_SRC_INC 573 | DWC_CTLL_FC_M2M; 574 prev = first = NULL; 575 576 for (offset = 0; offset < len; offset += xfer_count) { 577 desc = dwc_desc_get(dwc); 578 if (!desc) 579 goto err_desc_get; 580 581 ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count); 582 583 lli_write(desc, sar, src + offset); 584 lli_write(desc, dar, dest + offset); 585 lli_write(desc, ctllo, ctllo); 586 lli_write(desc, ctlhi, ctlhi); 587 desc->len = xfer_count; 588 589 if (!first) { 590 first = desc; 591 } else { 592 lli_write(prev, llp, desc->txd.phys | lms); 593 list_add_tail(&desc->desc_node, &first->tx_list); 594 } 595 prev = desc; 596 } 597 598 if (flags & DMA_PREP_INTERRUPT) 599 /* Trigger interrupt after last block */ 600 lli_set(prev, ctllo, DWC_CTLL_INT_EN); 601 602 prev->lli.llp = 0; 603 lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN); 604 first->txd.flags = flags; 605 first->total_len = len; 606 607 return &first->txd; 608 609 err_desc_get: 610 dwc_desc_put(dwc, first); 611 return NULL; 612 } 613 614 static struct dma_async_tx_descriptor * 615 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 616 unsigned int sg_len, enum dma_transfer_direction direction, 617 unsigned long flags, void *context) 618 { 619 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 620 struct dw_dma *dw = to_dw_dma(chan->device); 621 struct dma_slave_config *sconfig = &dwc->dma_sconfig; 622 struct dw_desc *prev; 623 struct dw_desc *first; 624 u32 ctllo, ctlhi; 625 u8 lms = DWC_LLP_LMS(dwc->dws.m_master); 626 dma_addr_t reg; 627 unsigned int reg_width; 628 unsigned int mem_width; 629 unsigned int i; 630 struct scatterlist *sg; 631 size_t total_len = 0; 632 633 dev_vdbg(chan2dev(chan), "%s\n", __func__); 634 635 if (unlikely(!is_slave_direction(direction) || !sg_len)) 636 return NULL; 637 638 dwc->direction = direction; 639 640 prev = first = NULL; 641 642 switch (direction) { 643 case DMA_MEM_TO_DEV: 644 reg_width = __ffs(sconfig->dst_addr_width); 645 reg = sconfig->dst_addr; 646 ctllo = dw->prepare_ctllo(dwc) 647 | DWC_CTLL_DST_WIDTH(reg_width) 648 | DWC_CTLL_DST_FIX 649 | DWC_CTLL_SRC_INC; 650 651 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) : 652 DWC_CTLL_FC(DW_DMA_FC_D_M2P); 653 654 for_each_sg(sgl, sg, sg_len, i) { 655 struct dw_desc *desc; 656 u32 len, mem; 657 size_t dlen; 658 659 mem = sg_dma_address(sg); 660 len = sg_dma_len(sg); 661 662 mem_width = __ffs(sconfig->src_addr_width | mem | len); 663 664 slave_sg_todev_fill_desc: 665 desc = dwc_desc_get(dwc); 666 if (!desc) 667 goto err_desc_get; 668 669 ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen); 670 671 lli_write(desc, sar, mem); 672 lli_write(desc, dar, reg); 673 lli_write(desc, ctlhi, ctlhi); 674 lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width)); 675 desc->len = dlen; 676 677 if (!first) { 678 first = desc; 679 } else { 680 lli_write(prev, llp, desc->txd.phys | lms); 681 list_add_tail(&desc->desc_node, &first->tx_list); 682 } 683 prev = desc; 684 685 mem += dlen; 686 len -= dlen; 687 total_len += dlen; 688 689 if (len) 690 goto slave_sg_todev_fill_desc; 691 } 692 break; 693 case DMA_DEV_TO_MEM: 694 reg_width = __ffs(sconfig->src_addr_width); 695 reg = sconfig->src_addr; 696 ctllo = dw->prepare_ctllo(dwc) 697 | DWC_CTLL_SRC_WIDTH(reg_width) 698 | DWC_CTLL_DST_INC 699 | DWC_CTLL_SRC_FIX; 700 701 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) : 702 DWC_CTLL_FC(DW_DMA_FC_D_P2M); 703 704 for_each_sg(sgl, sg, sg_len, i) { 705 struct dw_desc *desc; 706 u32 len, mem; 707 size_t dlen; 708 709 mem = sg_dma_address(sg); 710 len = sg_dma_len(sg); 711 712 slave_sg_fromdev_fill_desc: 713 desc = dwc_desc_get(dwc); 714 if (!desc) 715 goto err_desc_get; 716 717 ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen); 718 719 lli_write(desc, sar, reg); 720 lli_write(desc, dar, mem); 721 lli_write(desc, ctlhi, ctlhi); 722 mem_width = __ffs(sconfig->dst_addr_width | mem); 723 lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); 724 desc->len = dlen; 725 726 if (!first) { 727 first = desc; 728 } else { 729 lli_write(prev, llp, desc->txd.phys | lms); 730 list_add_tail(&desc->desc_node, &first->tx_list); 731 } 732 prev = desc; 733 734 mem += dlen; 735 len -= dlen; 736 total_len += dlen; 737 738 if (len) 739 goto slave_sg_fromdev_fill_desc; 740 } 741 break; 742 default: 743 return NULL; 744 } 745 746 if (flags & DMA_PREP_INTERRUPT) 747 /* Trigger interrupt after last block */ 748 lli_set(prev, ctllo, DWC_CTLL_INT_EN); 749 750 prev->lli.llp = 0; 751 lli_clear(prev, ctllo, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN); 752 first->total_len = total_len; 753 754 return &first->txd; 755 756 err_desc_get: 757 dev_err(chan2dev(chan), 758 "not enough descriptors available. Direction %d\n", direction); 759 dwc_desc_put(dwc, first); 760 return NULL; 761 } 762 763 bool dw_dma_filter(struct dma_chan *chan, void *param) 764 { 765 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 766 struct dw_dma_slave *dws = param; 767 768 if (dws->dma_dev != chan->device->dev) 769 return false; 770 771 /* permit channels in accordance with the channels mask */ 772 if (dws->channels && !(dws->channels & dwc->mask)) 773 return false; 774 775 /* We have to copy data since dws can be temporary storage */ 776 memcpy(&dwc->dws, dws, sizeof(struct dw_dma_slave)); 777 778 return true; 779 } 780 EXPORT_SYMBOL_GPL(dw_dma_filter); 781 782 static int dwc_verify_maxburst(struct dma_chan *chan) 783 { 784 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 785 786 dwc->dma_sconfig.src_maxburst = 787 clamp(dwc->dma_sconfig.src_maxburst, 1U, dwc->max_burst); 788 dwc->dma_sconfig.dst_maxburst = 789 clamp(dwc->dma_sconfig.dst_maxburst, 1U, dwc->max_burst); 790 791 dwc->dma_sconfig.src_maxburst = 792 rounddown_pow_of_two(dwc->dma_sconfig.src_maxburst); 793 dwc->dma_sconfig.dst_maxburst = 794 rounddown_pow_of_two(dwc->dma_sconfig.dst_maxburst); 795 796 return 0; 797 } 798 799 static int dwc_verify_p_buswidth(struct dma_chan *chan) 800 { 801 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 802 struct dw_dma *dw = to_dw_dma(chan->device); 803 u32 reg_width, max_width; 804 805 if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) 806 reg_width = dwc->dma_sconfig.dst_addr_width; 807 else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) 808 reg_width = dwc->dma_sconfig.src_addr_width; 809 else /* DMA_MEM_TO_MEM */ 810 return 0; 811 812 max_width = dw->pdata->data_width[dwc->dws.p_master]; 813 814 /* Fall-back to 1-byte transfer width if undefined */ 815 if (reg_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 816 reg_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 817 else if (!is_power_of_2(reg_width) || reg_width > max_width) 818 return -EINVAL; 819 else /* bus width is valid */ 820 return 0; 821 822 /* Update undefined addr width value */ 823 if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) 824 dwc->dma_sconfig.dst_addr_width = reg_width; 825 else /* DMA_DEV_TO_MEM */ 826 dwc->dma_sconfig.src_addr_width = reg_width; 827 828 return 0; 829 } 830 831 static int dwc_verify_m_buswidth(struct dma_chan *chan) 832 { 833 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 834 struct dw_dma *dw = to_dw_dma(chan->device); 835 u32 reg_width, reg_burst, mem_width; 836 837 mem_width = dw->pdata->data_width[dwc->dws.m_master]; 838 839 /* 840 * It's possible to have a data portion locked in the DMA FIFO in case 841 * of the channel suspension. Subsequent channel disabling will cause 842 * that data silent loss. In order to prevent that maintain the src and 843 * dst transfer widths coherency by means of the relation: 844 * (CTLx.SRC_TR_WIDTH * CTLx.SRC_MSIZE >= CTLx.DST_TR_WIDTH) 845 * Look for the details in the commit message that brings this change. 846 * 847 * Note the DMA configs utilized in the calculations below must have 848 * been verified to have correct values by this method call. 849 */ 850 if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) { 851 reg_width = dwc->dma_sconfig.dst_addr_width; 852 if (mem_width < reg_width) 853 return -EINVAL; 854 855 dwc->dma_sconfig.src_addr_width = mem_width; 856 } else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) { 857 reg_width = dwc->dma_sconfig.src_addr_width; 858 reg_burst = dwc->dma_sconfig.src_maxburst; 859 860 dwc->dma_sconfig.dst_addr_width = min(mem_width, reg_width * reg_burst); 861 } 862 863 return 0; 864 } 865 866 static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) 867 { 868 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 869 int ret; 870 871 memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); 872 873 ret = dwc_verify_maxburst(chan); 874 if (ret) 875 return ret; 876 877 ret = dwc_verify_p_buswidth(chan); 878 if (ret) 879 return ret; 880 881 ret = dwc_verify_m_buswidth(chan); 882 if (ret) 883 return ret; 884 885 return 0; 886 } 887 888 static void dwc_chan_pause(struct dw_dma_chan *dwc, bool drain) 889 { 890 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 891 unsigned int count = 20; /* timeout iterations */ 892 893 dw->suspend_chan(dwc, drain); 894 895 while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--) 896 udelay(2); 897 898 set_bit(DW_DMA_IS_PAUSED, &dwc->flags); 899 } 900 901 static int dwc_pause(struct dma_chan *chan) 902 { 903 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 904 unsigned long flags; 905 906 spin_lock_irqsave(&dwc->lock, flags); 907 dwc_chan_pause(dwc, false); 908 spin_unlock_irqrestore(&dwc->lock, flags); 909 910 return 0; 911 } 912 913 static inline void dwc_chan_resume(struct dw_dma_chan *dwc, bool drain) 914 { 915 struct dw_dma *dw = to_dw_dma(dwc->chan.device); 916 917 dw->resume_chan(dwc, drain); 918 919 clear_bit(DW_DMA_IS_PAUSED, &dwc->flags); 920 } 921 922 static int dwc_resume(struct dma_chan *chan) 923 { 924 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 925 unsigned long flags; 926 927 spin_lock_irqsave(&dwc->lock, flags); 928 929 if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) 930 dwc_chan_resume(dwc, false); 931 932 spin_unlock_irqrestore(&dwc->lock, flags); 933 934 return 0; 935 } 936 937 static int dwc_terminate_all(struct dma_chan *chan) 938 { 939 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 940 struct dw_dma *dw = to_dw_dma(chan->device); 941 struct dw_desc *desc, *_desc; 942 unsigned long flags; 943 LIST_HEAD(list); 944 945 spin_lock_irqsave(&dwc->lock, flags); 946 947 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags); 948 949 dwc_chan_pause(dwc, true); 950 951 dwc_chan_disable(dw, dwc); 952 953 dwc_chan_resume(dwc, true); 954 955 /* active_list entries will end up before queued entries */ 956 list_splice_init(&dwc->queue, &list); 957 list_splice_init(&dwc->active_list, &list); 958 959 spin_unlock_irqrestore(&dwc->lock, flags); 960 961 /* Flush all pending and queued descriptors */ 962 list_for_each_entry_safe(desc, _desc, &list, desc_node) 963 dwc_descriptor_complete(dwc, desc, false); 964 965 return 0; 966 } 967 968 static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c) 969 { 970 struct dw_desc *desc; 971 972 list_for_each_entry(desc, &dwc->active_list, desc_node) 973 if (desc->txd.cookie == c) 974 return desc; 975 976 return NULL; 977 } 978 979 static u32 dwc_get_residue_and_status(struct dw_dma_chan *dwc, dma_cookie_t cookie, 980 enum dma_status *status) 981 { 982 struct dw_desc *desc; 983 unsigned long flags; 984 u32 residue; 985 986 spin_lock_irqsave(&dwc->lock, flags); 987 988 desc = dwc_find_desc(dwc, cookie); 989 if (desc) { 990 if (desc == dwc_first_active(dwc)) { 991 residue = desc->residue; 992 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue) 993 residue -= dwc_get_sent(dwc); 994 if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) 995 *status = DMA_PAUSED; 996 } else { 997 residue = desc->total_len; 998 } 999 } else { 1000 residue = 0; 1001 } 1002 1003 spin_unlock_irqrestore(&dwc->lock, flags); 1004 return residue; 1005 } 1006 1007 static enum dma_status 1008 dwc_tx_status(struct dma_chan *chan, 1009 dma_cookie_t cookie, 1010 struct dma_tx_state *txstate) 1011 { 1012 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 1013 enum dma_status ret; 1014 1015 ret = dma_cookie_status(chan, cookie, txstate); 1016 if (ret == DMA_COMPLETE) 1017 return ret; 1018 1019 dwc_scan_descriptors(to_dw_dma(chan->device), dwc); 1020 1021 ret = dma_cookie_status(chan, cookie, txstate); 1022 if (ret == DMA_COMPLETE) 1023 return ret; 1024 1025 dma_set_residue(txstate, dwc_get_residue_and_status(dwc, cookie, &ret)); 1026 return ret; 1027 } 1028 1029 static void dwc_issue_pending(struct dma_chan *chan) 1030 { 1031 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 1032 unsigned long flags; 1033 1034 spin_lock_irqsave(&dwc->lock, flags); 1035 if (list_empty(&dwc->active_list)) 1036 dwc_dostart_first_queued(dwc); 1037 spin_unlock_irqrestore(&dwc->lock, flags); 1038 } 1039 1040 /*----------------------------------------------------------------------*/ 1041 1042 void do_dw_dma_off(struct dw_dma *dw) 1043 { 1044 dma_writel(dw, CFG, 0); 1045 1046 channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask); 1047 channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask); 1048 channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask); 1049 channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask); 1050 channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask); 1051 1052 while (dma_readl(dw, CFG) & DW_CFG_DMA_EN) 1053 cpu_relax(); 1054 } 1055 1056 void do_dw_dma_on(struct dw_dma *dw) 1057 { 1058 dma_writel(dw, CFG, DW_CFG_DMA_EN); 1059 } 1060 1061 static int dwc_alloc_chan_resources(struct dma_chan *chan) 1062 { 1063 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 1064 struct dw_dma *dw = to_dw_dma(chan->device); 1065 1066 dev_vdbg(chan2dev(chan), "%s\n", __func__); 1067 1068 /* ASSERT: channel is idle */ 1069 if (dma_readl(dw, CH_EN) & dwc->mask) { 1070 dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); 1071 return -EIO; 1072 } 1073 1074 dma_cookie_init(chan); 1075 1076 /* 1077 * NOTE: some controllers may have additional features that we 1078 * need to initialize here, like "scatter-gather" (which 1079 * doesn't mean what you think it means), and status writeback. 1080 */ 1081 1082 /* 1083 * We need controller-specific data to set up slave transfers. 1084 */ 1085 if (chan->private && !dw_dma_filter(chan, chan->private)) { 1086 dev_warn(chan2dev(chan), "Wrong controller-specific data\n"); 1087 return -EINVAL; 1088 } 1089 1090 /* Enable controller here if needed */ 1091 if (!dw->in_use) 1092 do_dw_dma_on(dw); 1093 dw->in_use |= dwc->mask; 1094 1095 return 0; 1096 } 1097 1098 static void dwc_free_chan_resources(struct dma_chan *chan) 1099 { 1100 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 1101 struct dw_dma *dw = to_dw_dma(chan->device); 1102 unsigned long flags; 1103 1104 dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__, 1105 dwc->descs_allocated); 1106 1107 /* ASSERT: channel is idle */ 1108 BUG_ON(!list_empty(&dwc->active_list)); 1109 BUG_ON(!list_empty(&dwc->queue)); 1110 BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask); 1111 1112 spin_lock_irqsave(&dwc->lock, flags); 1113 1114 /* Clear custom channel configuration */ 1115 memset(&dwc->dws, 0, sizeof(struct dw_dma_slave)); 1116 1117 /* Disable interrupts */ 1118 channel_clear_bit(dw, MASK.XFER, dwc->mask); 1119 channel_clear_bit(dw, MASK.BLOCK, dwc->mask); 1120 channel_clear_bit(dw, MASK.ERROR, dwc->mask); 1121 1122 spin_unlock_irqrestore(&dwc->lock, flags); 1123 1124 /* Disable controller in case it was a last user */ 1125 dw->in_use &= ~dwc->mask; 1126 if (!dw->in_use) 1127 do_dw_dma_off(dw); 1128 1129 dev_vdbg(chan2dev(chan), "%s: done\n", __func__); 1130 } 1131 1132 static void dwc_caps(struct dma_chan *chan, struct dma_slave_caps *caps) 1133 { 1134 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 1135 1136 caps->max_burst = dwc->max_burst; 1137 1138 /* 1139 * It might be crucial for some devices to have the hardware 1140 * accelerated multi-block transfers supported, aka LLPs in DW DMAC 1141 * notation. So if LLPs are supported then max_sg_burst is set to 1142 * zero which means unlimited number of SG entries can be handled in a 1143 * single DMA transaction, otherwise it's just one SG entry. 1144 */ 1145 if (dwc->nollp) 1146 caps->max_sg_burst = 1; 1147 else 1148 caps->max_sg_burst = 0; 1149 } 1150 1151 int do_dma_probe(struct dw_dma_chip *chip) 1152 { 1153 struct dw_dma *dw = chip->dw; 1154 struct dw_dma_platform_data *pdata; 1155 bool autocfg = false; 1156 unsigned int dw_params; 1157 unsigned int i; 1158 int ret; 1159 1160 dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL); 1161 if (!dw->pdata) 1162 return -ENOMEM; 1163 1164 dw->regs = chip->regs; 1165 1166 pm_runtime_get_sync(chip->dev); 1167 1168 if (!chip->pdata) { 1169 dw_params = dma_readl(dw, DW_PARAMS); 1170 dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params); 1171 1172 autocfg = dw_params >> DW_PARAMS_EN & 1; 1173 if (!autocfg) { 1174 ret = -EINVAL; 1175 goto err_pdata; 1176 } 1177 1178 /* Reassign the platform data pointer */ 1179 pdata = dw->pdata; 1180 1181 /* Get hardware configuration parameters */ 1182 pdata->nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 7) + 1; 1183 pdata->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1; 1184 for (i = 0; i < pdata->nr_masters; i++) { 1185 pdata->data_width[i] = 1186 4 << (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3); 1187 } 1188 pdata->block_size = dma_readl(dw, MAX_BLK_SIZE); 1189 1190 /* Fill platform data with the default values */ 1191 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; 1192 pdata->chan_priority = CHAN_PRIORITY_ASCENDING; 1193 } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { 1194 ret = -EINVAL; 1195 goto err_pdata; 1196 } else { 1197 memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata)); 1198 1199 /* Reassign the platform data pointer */ 1200 pdata = dw->pdata; 1201 } 1202 1203 dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan), 1204 GFP_KERNEL); 1205 if (!dw->chan) { 1206 ret = -ENOMEM; 1207 goto err_pdata; 1208 } 1209 1210 /* Calculate all channel mask before DMA setup */ 1211 dw->all_chan_mask = (1 << pdata->nr_channels) - 1; 1212 1213 /* Force dma off, just in case */ 1214 dw->disable(dw); 1215 1216 /* Device and instance ID for IRQ and DMA pool */ 1217 dw->set_device_name(dw, chip->id); 1218 1219 /* Create a pool of consistent memory blocks for hardware descriptors */ 1220 dw->desc_pool = dmam_pool_create(dw->name, chip->dev, 1221 sizeof(struct dw_desc), 4, 0); 1222 if (!dw->desc_pool) { 1223 dev_err(chip->dev, "No memory for descriptors dma pool\n"); 1224 ret = -ENOMEM; 1225 goto err_pdata; 1226 } 1227 1228 tasklet_setup(&dw->tasklet, dw_dma_tasklet); 1229 1230 ret = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, 1231 dw->name, dw); 1232 if (ret) 1233 goto err_pdata; 1234 1235 INIT_LIST_HEAD(&dw->dma.channels); 1236 for (i = 0; i < pdata->nr_channels; i++) { 1237 struct dw_dma_chan *dwc = &dw->chan[i]; 1238 1239 dwc->chan.device = &dw->dma; 1240 dma_cookie_init(&dwc->chan); 1241 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING) 1242 list_add_tail(&dwc->chan.device_node, 1243 &dw->dma.channels); 1244 else 1245 list_add(&dwc->chan.device_node, &dw->dma.channels); 1246 1247 /* 7 is highest priority & 0 is lowest. */ 1248 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING) 1249 dwc->priority = pdata->nr_channels - i - 1; 1250 else 1251 dwc->priority = i; 1252 1253 dwc->ch_regs = &__dw_regs(dw)->CHAN[i]; 1254 spin_lock_init(&dwc->lock); 1255 dwc->mask = 1 << i; 1256 1257 INIT_LIST_HEAD(&dwc->active_list); 1258 INIT_LIST_HEAD(&dwc->queue); 1259 1260 channel_clear_bit(dw, CH_EN, dwc->mask); 1261 1262 dwc->direction = DMA_TRANS_NONE; 1263 1264 /* Hardware configuration */ 1265 if (autocfg) { 1266 unsigned int r = DW_DMA_MAX_NR_CHANNELS - i - 1; 1267 void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r]; 1268 unsigned int dwc_params = readl(addr); 1269 1270 dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i, 1271 dwc_params); 1272 1273 /* 1274 * Decode maximum block size for given channel. The 1275 * stored 4 bit value represents blocks from 0x00 for 3 1276 * up to 0x0a for 4095. 1277 */ 1278 dwc->block_size = 1279 (4 << ((pdata->block_size >> 4 * i) & 0xf)) - 1; 1280 1281 /* 1282 * According to the DW DMA databook the true scatter- 1283 * gether LLPs aren't available if either multi-block 1284 * config is disabled (CHx_MULTI_BLK_EN == 0) or the 1285 * LLP register is hard-coded to zeros 1286 * (CHx_HC_LLP == 1). 1287 */ 1288 dwc->nollp = 1289 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0 || 1290 (dwc_params >> DWC_PARAMS_HC_LLP & 0x1) == 1; 1291 dwc->max_burst = 1292 (0x4 << (dwc_params >> DWC_PARAMS_MSIZE & 0x7)); 1293 } else { 1294 dwc->block_size = pdata->block_size; 1295 dwc->nollp = !pdata->multi_block[i]; 1296 dwc->max_burst = pdata->max_burst[i] ?: DW_DMA_MAX_BURST; 1297 } 1298 } 1299 1300 /* Clear all interrupts on all channels. */ 1301 dma_writel(dw, CLEAR.XFER, dw->all_chan_mask); 1302 dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask); 1303 dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask); 1304 dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask); 1305 dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask); 1306 1307 /* Set capabilities */ 1308 dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); 1309 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask); 1310 dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); 1311 1312 dw->dma.dev = chip->dev; 1313 dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources; 1314 dw->dma.device_free_chan_resources = dwc_free_chan_resources; 1315 1316 dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy; 1317 dw->dma.device_prep_slave_sg = dwc_prep_slave_sg; 1318 1319 dw->dma.device_caps = dwc_caps; 1320 dw->dma.device_config = dwc_config; 1321 dw->dma.device_pause = dwc_pause; 1322 dw->dma.device_resume = dwc_resume; 1323 dw->dma.device_terminate_all = dwc_terminate_all; 1324 1325 dw->dma.device_tx_status = dwc_tx_status; 1326 dw->dma.device_issue_pending = dwc_issue_pending; 1327 1328 /* DMA capabilities */ 1329 dw->dma.min_burst = DW_DMA_MIN_BURST; 1330 dw->dma.max_burst = DW_DMA_MAX_BURST; 1331 dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS; 1332 dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS; 1333 dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | 1334 BIT(DMA_MEM_TO_MEM); 1335 dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1336 1337 /* 1338 * For now there is no hardware with non uniform maximum block size 1339 * across all of the device channels, so we set the maximum segment 1340 * size as the block size found for the very first channel. 1341 */ 1342 dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size); 1343 1344 ret = dma_async_device_register(&dw->dma); 1345 if (ret) 1346 goto err_dma_register; 1347 1348 dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", 1349 pdata->nr_channels); 1350 1351 pm_runtime_put_sync_suspend(chip->dev); 1352 1353 return 0; 1354 1355 err_dma_register: 1356 free_irq(chip->irq, dw); 1357 err_pdata: 1358 pm_runtime_put_sync_suspend(chip->dev); 1359 return ret; 1360 } 1361 1362 int do_dma_remove(struct dw_dma_chip *chip) 1363 { 1364 struct dw_dma *dw = chip->dw; 1365 struct dw_dma_chan *dwc, *_dwc; 1366 1367 pm_runtime_get_sync(chip->dev); 1368 1369 do_dw_dma_off(dw); 1370 dma_async_device_unregister(&dw->dma); 1371 1372 free_irq(chip->irq, dw); 1373 tasklet_kill(&dw->tasklet); 1374 1375 list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels, 1376 chan.device_node) { 1377 list_del(&dwc->chan.device_node); 1378 channel_clear_bit(dw, CH_EN, dwc->mask); 1379 } 1380 1381 pm_runtime_put_sync_suspend(chip->dev); 1382 return 0; 1383 } 1384 1385 int do_dw_dma_disable(struct dw_dma_chip *chip) 1386 { 1387 struct dw_dma *dw = chip->dw; 1388 1389 dw->disable(dw); 1390 return 0; 1391 } 1392 EXPORT_SYMBOL_GPL(do_dw_dma_disable); 1393 1394 int do_dw_dma_enable(struct dw_dma_chip *chip) 1395 { 1396 struct dw_dma *dw = chip->dw; 1397 1398 dw->enable(dw); 1399 return 0; 1400 } 1401 EXPORT_SYMBOL_GPL(do_dw_dma_enable); 1402 1403 MODULE_LICENSE("GPL v2"); 1404 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver"); 1405 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 1406 MODULE_AUTHOR("Viresh Kumar <vireshk@kernel.org>"); 1407