1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2018-2019 MediaTek Inc. 3 4 /* 5 * Driver for MediaTek Command-Queue DMA Controller 6 * 7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com> 8 * 9 */ 10 11 #include <linux/bitops.h> 12 #include <linux/clk.h> 13 #include <linux/dmaengine.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/err.h> 16 #include <linux/iopoll.h> 17 #include <linux/interrupt.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_dma.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_runtime.h> 24 #include <linux/refcount.h> 25 #include <linux/slab.h> 26 27 #include "../virt-dma.h" 28 29 #define MTK_CQDMA_USEC_POLL 10 30 #define MTK_CQDMA_TIMEOUT_POLL 1000 31 #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 32 #define MTK_CQDMA_ALIGN_SIZE 1 33 34 /* The default number of virtual channel */ 35 #define MTK_CQDMA_NR_VCHANS 32 36 37 /* The default number of physical channel */ 38 #define MTK_CQDMA_NR_PCHANS 3 39 40 /* Registers for underlying dma manipulation */ 41 #define MTK_CQDMA_INT_FLAG 0x0 42 #define MTK_CQDMA_INT_EN 0x4 43 #define MTK_CQDMA_EN 0x8 44 #define MTK_CQDMA_RESET 0xc 45 #define MTK_CQDMA_FLUSH 0x14 46 #define MTK_CQDMA_SRC 0x1c 47 #define MTK_CQDMA_DST 0x20 48 #define MTK_CQDMA_LEN1 0x24 49 #define MTK_CQDMA_LEN2 0x28 50 #define MTK_CQDMA_SRC2 0x60 51 #define MTK_CQDMA_DST2 0x64 52 53 /* Registers setting */ 54 #define MTK_CQDMA_EN_BIT BIT(0) 55 #define MTK_CQDMA_INT_FLAG_BIT BIT(0) 56 #define MTK_CQDMA_INT_EN_BIT BIT(0) 57 #define MTK_CQDMA_FLUSH_BIT BIT(0) 58 59 #define MTK_CQDMA_WARM_RST_BIT BIT(0) 60 #define MTK_CQDMA_HARD_RST_BIT BIT(1) 61 62 #define MTK_CQDMA_MAX_LEN GENMASK(27, 0) 63 #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0) 64 #define MTK_CQDMA_ADDR2_SHFIT (32) 65 66 /** 67 * struct mtk_cqdma_vdesc - The struct holding info describing virtual 68 * descriptor (CVD) 69 * @vd: An instance for struct virt_dma_desc 70 * @len: The total data size device wants to move 71 * @residue: The remaining data size device will move 72 * @dest: The destination address device wants to move to 73 * @src: The source address device wants to move from 74 * @ch: The pointer to the corresponding dma channel 75 * @node: The lise_head struct to build link-list for VDs 76 * @parent: The pointer to the parent CVD 77 */ 78 struct mtk_cqdma_vdesc { 79 struct virt_dma_desc vd; 80 size_t len; 81 size_t residue; 82 dma_addr_t dest; 83 dma_addr_t src; 84 struct dma_chan *ch; 85 86 struct list_head node; 87 struct mtk_cqdma_vdesc *parent; 88 }; 89 90 /** 91 * struct mtk_cqdma_pchan - The struct holding info describing physical 92 * channel (PC) 93 * @queue: Queue for the PDs issued to this PC 94 * @base: The mapped register I/O base of this PC 95 * @irq: The IRQ that this PC are using 96 * @refcnt: Track how many VCs are using this PC 97 * @tasklet: Tasklet for this PC 98 * @lock: Lock protect agaisting multiple VCs access PC 99 */ 100 struct mtk_cqdma_pchan { 101 struct list_head queue; 102 void __iomem *base; 103 u32 irq; 104 105 refcount_t refcnt; 106 107 struct tasklet_struct tasklet; 108 109 /* lock to protect PC */ 110 spinlock_t lock; 111 }; 112 113 /** 114 * struct mtk_cqdma_vchan - The struct holding info describing virtual 115 * channel (VC) 116 * @vc: An instance for struct virt_dma_chan 117 * @pc: The pointer to the underlying PC 118 * @issue_completion: The wait for all issued descriptors completited 119 * @issue_synchronize: Bool indicating channel synchronization starts 120 */ 121 struct mtk_cqdma_vchan { 122 struct virt_dma_chan vc; 123 struct mtk_cqdma_pchan *pc; 124 struct completion issue_completion; 125 bool issue_synchronize; 126 }; 127 128 /** 129 * struct mtk_cqdma_device - The struct holding info describing CQDMA 130 * device 131 * @ddev: An instance for struct dma_device 132 * @clk: The clock that device internal is using 133 * @dma_requests: The number of VCs the device supports to 134 * @dma_channels: The number of PCs the device supports to 135 * @vc: The pointer to all available VCs 136 * @pc: The pointer to all the underlying PCs 137 */ 138 struct mtk_cqdma_device { 139 struct dma_device ddev; 140 struct clk *clk; 141 142 u32 dma_requests; 143 u32 dma_channels; 144 struct mtk_cqdma_vchan *vc; 145 struct mtk_cqdma_pchan **pc; 146 }; 147 148 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan) 149 { 150 return container_of(chan->device, struct mtk_cqdma_device, ddev); 151 } 152 153 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan) 154 { 155 return container_of(chan, struct mtk_cqdma_vchan, vc.chan); 156 } 157 158 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd) 159 { 160 return container_of(vd, struct mtk_cqdma_vdesc, vd); 161 } 162 163 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma) 164 { 165 return cqdma->ddev.dev; 166 } 167 168 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg) 169 { 170 return readl(pc->base + reg); 171 } 172 173 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 174 { 175 writel_relaxed(val, pc->base + reg); 176 } 177 178 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg, 179 u32 mask, u32 set) 180 { 181 u32 val; 182 183 val = mtk_dma_read(pc, reg); 184 val &= ~mask; 185 val |= set; 186 mtk_dma_write(pc, reg, val); 187 } 188 189 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 190 { 191 mtk_dma_rmw(pc, reg, 0, val); 192 } 193 194 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) 195 { 196 mtk_dma_rmw(pc, reg, val, 0); 197 } 198 199 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd) 200 { 201 kfree(to_cqdma_vdesc(vd)); 202 } 203 204 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic) 205 { 206 u32 status = 0; 207 208 if (!atomic) 209 return readl_poll_timeout(pc->base + MTK_CQDMA_EN, 210 status, 211 !(status & MTK_CQDMA_EN_BIT), 212 MTK_CQDMA_USEC_POLL, 213 MTK_CQDMA_TIMEOUT_POLL); 214 215 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN, 216 status, 217 !(status & MTK_CQDMA_EN_BIT), 218 MTK_CQDMA_USEC_POLL, 219 MTK_CQDMA_TIMEOUT_POLL); 220 } 221 222 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc) 223 { 224 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); 225 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); 226 227 return mtk_cqdma_poll_engine_done(pc, true); 228 } 229 230 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc, 231 struct mtk_cqdma_vdesc *cvd) 232 { 233 /* wait for the previous transaction done */ 234 if (mtk_cqdma_poll_engine_done(pc, true) < 0) 235 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n"); 236 237 /* warm reset the dma engine for the new transaction */ 238 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT); 239 if (mtk_cqdma_poll_engine_done(pc, true) < 0) 240 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n"); 241 242 /* setup the source */ 243 mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT); 244 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 245 mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT); 246 #else 247 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0); 248 #endif 249 250 /* setup the destination */ 251 mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT); 252 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 253 mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT); 254 #else 255 mtk_dma_set(pc, MTK_CQDMA_DST2, 0); 256 #endif 257 258 /* setup the length */ 259 mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len); 260 261 /* start dma engine */ 262 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT); 263 } 264 265 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc) 266 { 267 struct virt_dma_desc *vd, *vd2; 268 struct mtk_cqdma_pchan *pc = cvc->pc; 269 struct mtk_cqdma_vdesc *cvd; 270 bool trigger_engine = false; 271 272 lockdep_assert_held(&cvc->vc.lock); 273 lockdep_assert_held(&pc->lock); 274 275 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) { 276 /* need to trigger dma engine if PC's queue is empty */ 277 if (list_empty(&pc->queue)) 278 trigger_engine = true; 279 280 cvd = to_cqdma_vdesc(vd); 281 282 /* add VD into PC's queue */ 283 list_add_tail(&cvd->node, &pc->queue); 284 285 /* start the dma engine */ 286 if (trigger_engine) 287 mtk_cqdma_start(pc, cvd); 288 289 /* remove VD from list desc_issued */ 290 list_del(&vd->node); 291 } 292 } 293 294 /* 295 * return true if this VC is active, 296 * meaning that there are VDs under processing by the PC 297 */ 298 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc) 299 { 300 struct mtk_cqdma_vdesc *cvd; 301 302 list_for_each_entry(cvd, &cvc->pc->queue, node) 303 if (cvc == to_cqdma_vchan(cvd->ch)) 304 return true; 305 306 return false; 307 } 308 309 /* 310 * return the pointer of the CVD that is just consumed by the PC 311 */ 312 static struct mtk_cqdma_vdesc 313 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc) 314 { 315 struct mtk_cqdma_vchan *cvc; 316 struct mtk_cqdma_vdesc *cvd, *ret = NULL; 317 318 /* consume a CVD from PC's queue */ 319 cvd = list_first_entry_or_null(&pc->queue, 320 struct mtk_cqdma_vdesc, node); 321 if (unlikely(!cvd || !cvd->parent)) 322 return NULL; 323 324 cvc = to_cqdma_vchan(cvd->ch); 325 ret = cvd; 326 327 /* update residue of the parent CVD */ 328 cvd->parent->residue -= cvd->len; 329 330 /* delete CVD from PC's queue */ 331 list_del(&cvd->node); 332 333 spin_lock(&cvc->vc.lock); 334 335 /* check whether all the child CVDs completed */ 336 if (!cvd->parent->residue) { 337 /* add the parent VD into list desc_completed */ 338 vchan_cookie_complete(&cvd->parent->vd); 339 340 /* setup completion if this VC is under synchronization */ 341 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) { 342 complete(&cvc->issue_completion); 343 cvc->issue_synchronize = false; 344 } 345 } 346 347 spin_unlock(&cvc->vc.lock); 348 349 /* start transaction for next CVD in the queue */ 350 cvd = list_first_entry_or_null(&pc->queue, 351 struct mtk_cqdma_vdesc, node); 352 if (cvd) 353 mtk_cqdma_start(pc, cvd); 354 355 return ret; 356 } 357 358 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t) 359 { 360 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet); 361 struct mtk_cqdma_vdesc *cvd = NULL; 362 unsigned long flags; 363 364 spin_lock_irqsave(&pc->lock, flags); 365 /* consume the queue */ 366 cvd = mtk_cqdma_consume_work_queue(pc); 367 spin_unlock_irqrestore(&pc->lock, flags); 368 369 /* submit the next CVD */ 370 if (cvd) { 371 dma_run_dependencies(&cvd->vd.tx); 372 373 /* 374 * free child CVD after completion. 375 * the parent CVD would be freed with desc_free by user. 376 */ 377 if (cvd->parent != cvd) 378 kfree(cvd); 379 } 380 381 /* re-enable interrupt before leaving tasklet */ 382 enable_irq(pc->irq); 383 } 384 385 static irqreturn_t mtk_cqdma_irq(int irq, void *devid) 386 { 387 struct mtk_cqdma_device *cqdma = devid; 388 irqreturn_t ret = IRQ_NONE; 389 bool schedule_tasklet = false; 390 u32 i; 391 392 /* clear interrupt flags for each PC */ 393 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) { 394 spin_lock(&cqdma->pc[i]->lock); 395 if (mtk_dma_read(cqdma->pc[i], 396 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) { 397 /* clear interrupt */ 398 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG, 399 MTK_CQDMA_INT_FLAG_BIT); 400 401 schedule_tasklet = true; 402 ret = IRQ_HANDLED; 403 } 404 spin_unlock(&cqdma->pc[i]->lock); 405 406 if (schedule_tasklet) { 407 /* disable interrupt */ 408 disable_irq_nosync(cqdma->pc[i]->irq); 409 410 /* schedule the tasklet to handle the transactions */ 411 tasklet_schedule(&cqdma->pc[i]->tasklet); 412 } 413 } 414 415 return ret; 416 } 417 418 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, 419 dma_cookie_t cookie) 420 { 421 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 422 struct virt_dma_desc *vd; 423 424 list_for_each_entry(vd, &cvc->pc->queue, node) 425 if (vd->tx.cookie == cookie) { 426 return vd; 427 } 428 429 list_for_each_entry(vd, &cvc->vc.desc_issued, node) 430 if (vd->tx.cookie == cookie) 431 return vd; 432 433 return NULL; 434 } 435 436 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, 437 dma_cookie_t cookie, 438 struct dma_tx_state *txstate) 439 { 440 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 441 struct mtk_cqdma_vdesc *cvd; 442 struct virt_dma_desc *vd; 443 enum dma_status ret; 444 unsigned long flags; 445 size_t bytes = 0; 446 447 ret = dma_cookie_status(c, cookie, txstate); 448 if (ret == DMA_COMPLETE || !txstate) 449 return ret; 450 451 spin_lock_irqsave(&cvc->pc->lock, flags); 452 spin_lock(&cvc->vc.lock); 453 vd = mtk_cqdma_find_active_desc(c, cookie); 454 spin_unlock(&cvc->vc.lock); 455 spin_unlock_irqrestore(&cvc->pc->lock, flags); 456 457 if (vd) { 458 cvd = to_cqdma_vdesc(vd); 459 bytes = cvd->residue; 460 } 461 462 dma_set_residue(txstate, bytes); 463 464 return ret; 465 } 466 467 static void mtk_cqdma_issue_pending(struct dma_chan *c) 468 { 469 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 470 unsigned long pc_flags; 471 unsigned long vc_flags; 472 473 /* acquire PC's lock before VS's lock for lock dependency in tasklet */ 474 spin_lock_irqsave(&cvc->pc->lock, pc_flags); 475 spin_lock_irqsave(&cvc->vc.lock, vc_flags); 476 477 if (vchan_issue_pending(&cvc->vc)) 478 mtk_cqdma_issue_vchan_pending(cvc); 479 480 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); 481 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); 482 } 483 484 static struct dma_async_tx_descriptor * 485 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, 486 dma_addr_t src, size_t len, unsigned long flags) 487 { 488 struct mtk_cqdma_vdesc **cvd; 489 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL; 490 size_t i, tlen, nr_vd; 491 492 /* 493 * In the case that trsanction length is larger than the 494 * DMA engine supports, a single memcpy transaction needs 495 * to be separated into several DMA transactions. 496 * Each DMA transaction would be described by a CVD, 497 * and the first one is referred as the parent CVD, 498 * while the others are child CVDs. 499 * The parent CVD's tx descriptor is the only tx descriptor 500 * returned to the DMA user, and it should not be completed 501 * until all the child CVDs completed. 502 */ 503 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN); 504 cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT); 505 if (!cvd) 506 return NULL; 507 508 for (i = 0; i < nr_vd; ++i) { 509 cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT); 510 if (!cvd[i]) { 511 for (; i > 0; --i) 512 kfree(cvd[i - 1]); 513 return NULL; 514 } 515 516 /* setup dma channel */ 517 cvd[i]->ch = c; 518 519 /* setup source, destination, and length */ 520 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; 521 cvd[i]->len = tlen; 522 cvd[i]->src = src; 523 cvd[i]->dest = dest; 524 525 /* setup tx descriptor */ 526 tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags); 527 tx->next = NULL; 528 529 if (!i) { 530 cvd[0]->residue = len; 531 } else { 532 prev_tx->next = tx; 533 cvd[i]->residue = tlen; 534 } 535 536 cvd[i]->parent = cvd[0]; 537 538 /* update the src, dest, len, prev_tx for the next CVD */ 539 src += tlen; 540 dest += tlen; 541 len -= tlen; 542 prev_tx = tx; 543 } 544 545 return &cvd[0]->vd.tx; 546 } 547 548 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c) 549 { 550 struct virt_dma_chan *vc = to_virt_chan(c); 551 unsigned long flags; 552 LIST_HEAD(head); 553 554 /* 555 * set desc_allocated, desc_submitted, 556 * and desc_issued as the candicates to be freed 557 */ 558 spin_lock_irqsave(&vc->lock, flags); 559 list_splice_tail_init(&vc->desc_allocated, &head); 560 list_splice_tail_init(&vc->desc_submitted, &head); 561 list_splice_tail_init(&vc->desc_issued, &head); 562 spin_unlock_irqrestore(&vc->lock, flags); 563 564 /* free descriptor lists */ 565 vchan_dma_desc_free_list(vc, &head); 566 } 567 568 static void mtk_cqdma_free_active_desc(struct dma_chan *c) 569 { 570 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 571 bool sync_needed = false; 572 unsigned long pc_flags; 573 unsigned long vc_flags; 574 575 /* acquire PC's lock first due to lock dependency in dma ISR */ 576 spin_lock_irqsave(&cvc->pc->lock, pc_flags); 577 spin_lock_irqsave(&cvc->vc.lock, vc_flags); 578 579 /* synchronization is required if this VC is active */ 580 if (mtk_cqdma_is_vchan_active(cvc)) { 581 cvc->issue_synchronize = true; 582 sync_needed = true; 583 } 584 585 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags); 586 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags); 587 588 /* waiting for the completion of this VC */ 589 if (sync_needed) 590 wait_for_completion(&cvc->issue_completion); 591 592 /* free all descriptors in list desc_completed */ 593 vchan_synchronize(&cvc->vc); 594 595 WARN_ONCE(!list_empty(&cvc->vc.desc_completed), 596 "Desc pending still in list desc_completed\n"); 597 } 598 599 static int mtk_cqdma_terminate_all(struct dma_chan *c) 600 { 601 /* free descriptors not processed yet by hardware */ 602 mtk_cqdma_free_inactive_desc(c); 603 604 /* free descriptors being processed by hardware */ 605 mtk_cqdma_free_active_desc(c); 606 607 return 0; 608 } 609 610 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) 611 { 612 struct mtk_cqdma_device *cqdma = to_cqdma_dev(c); 613 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c); 614 struct mtk_cqdma_pchan *pc = NULL; 615 u32 i, min_refcnt = U32_MAX, refcnt; 616 unsigned long flags; 617 618 /* allocate PC with the minimum refcount */ 619 for (i = 0; i < cqdma->dma_channels; ++i) { 620 refcnt = refcount_read(&cqdma->pc[i]->refcnt); 621 if (refcnt < min_refcnt) { 622 pc = cqdma->pc[i]; 623 min_refcnt = refcnt; 624 } 625 } 626 627 if (!pc) 628 return -ENOSPC; 629 630 spin_lock_irqsave(&pc->lock, flags); 631 632 if (!refcount_read(&pc->refcnt)) { 633 /* allocate PC when the refcount is zero */ 634 mtk_cqdma_hard_reset(pc); 635 636 /* enable interrupt for this PC */ 637 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); 638 639 /* 640 * refcount_inc would complain increment on 0; use-after-free. 641 * Thus, we need to explicitly set it as 1 initially. 642 */ 643 refcount_set(&pc->refcnt, 1); 644 } else { 645 refcount_inc(&pc->refcnt); 646 } 647 648 spin_unlock_irqrestore(&pc->lock, flags); 649 650 vc->pc = pc; 651 652 return 0; 653 } 654 655 static void mtk_cqdma_free_chan_resources(struct dma_chan *c) 656 { 657 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); 658 unsigned long flags; 659 660 /* free all descriptors in all lists on the VC */ 661 mtk_cqdma_terminate_all(c); 662 663 spin_lock_irqsave(&cvc->pc->lock, flags); 664 665 /* PC is not freed until there is no VC mapped to it */ 666 if (refcount_dec_and_test(&cvc->pc->refcnt)) { 667 /* start the flush operation and stop the engine */ 668 mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); 669 670 /* wait for the completion of flush operation */ 671 if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0) 672 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n"); 673 674 /* clear the flush bit and interrupt flag */ 675 mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); 676 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG, 677 MTK_CQDMA_INT_FLAG_BIT); 678 679 /* disable interrupt for this PC */ 680 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); 681 } 682 683 spin_unlock_irqrestore(&cvc->pc->lock, flags); 684 } 685 686 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma) 687 { 688 unsigned long flags; 689 int err; 690 u32 i; 691 692 pm_runtime_enable(cqdma2dev(cqdma)); 693 pm_runtime_get_sync(cqdma2dev(cqdma)); 694 695 err = clk_prepare_enable(cqdma->clk); 696 697 if (err) { 698 pm_runtime_put_sync(cqdma2dev(cqdma)); 699 pm_runtime_disable(cqdma2dev(cqdma)); 700 return err; 701 } 702 703 /* reset all PCs */ 704 for (i = 0; i < cqdma->dma_channels; ++i) { 705 spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 706 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) { 707 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); 708 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 709 710 clk_disable_unprepare(cqdma->clk); 711 pm_runtime_put_sync(cqdma2dev(cqdma)); 712 pm_runtime_disable(cqdma2dev(cqdma)); 713 return -EINVAL; 714 } 715 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 716 } 717 718 return 0; 719 } 720 721 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma) 722 { 723 unsigned long flags; 724 u32 i; 725 726 /* reset all PCs */ 727 for (i = 0; i < cqdma->dma_channels; ++i) { 728 spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 729 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) 730 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n"); 731 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 732 } 733 734 clk_disable_unprepare(cqdma->clk); 735 736 pm_runtime_put_sync(cqdma2dev(cqdma)); 737 pm_runtime_disable(cqdma2dev(cqdma)); 738 } 739 740 static const struct of_device_id mtk_cqdma_match[] = { 741 { .compatible = "mediatek,mt6765-cqdma" }, 742 { /* sentinel */ } 743 }; 744 MODULE_DEVICE_TABLE(of, mtk_cqdma_match); 745 746 static int mtk_cqdma_probe(struct platform_device *pdev) 747 { 748 struct mtk_cqdma_device *cqdma; 749 struct mtk_cqdma_vchan *vc; 750 struct dma_device *dd; 751 int err; 752 u32 i; 753 754 cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL); 755 if (!cqdma) 756 return -ENOMEM; 757 758 dd = &cqdma->ddev; 759 760 cqdma->clk = devm_clk_get(&pdev->dev, "cqdma"); 761 if (IS_ERR(cqdma->clk)) { 762 dev_err(&pdev->dev, "No clock for %s\n", 763 dev_name(&pdev->dev)); 764 return PTR_ERR(cqdma->clk); 765 } 766 767 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 768 769 dd->copy_align = MTK_CQDMA_ALIGN_SIZE; 770 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources; 771 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources; 772 dd->device_tx_status = mtk_cqdma_tx_status; 773 dd->device_issue_pending = mtk_cqdma_issue_pending; 774 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy; 775 dd->device_terminate_all = mtk_cqdma_terminate_all; 776 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; 777 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; 778 dd->directions = BIT(DMA_MEM_TO_MEM); 779 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 780 dd->dev = &pdev->dev; 781 INIT_LIST_HEAD(&dd->channels); 782 783 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 784 "dma-requests", 785 &cqdma->dma_requests)) { 786 dev_info(&pdev->dev, 787 "Using %u as missing dma-requests property\n", 788 MTK_CQDMA_NR_VCHANS); 789 790 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS; 791 } 792 793 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 794 "dma-channels", 795 &cqdma->dma_channels)) { 796 dev_info(&pdev->dev, 797 "Using %u as missing dma-channels property\n", 798 MTK_CQDMA_NR_PCHANS); 799 800 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS; 801 } 802 803 cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels, 804 sizeof(*cqdma->pc), GFP_KERNEL); 805 if (!cqdma->pc) 806 return -ENOMEM; 807 808 /* initialization for PCs */ 809 for (i = 0; i < cqdma->dma_channels; ++i) { 810 cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1, 811 sizeof(**cqdma->pc), GFP_KERNEL); 812 if (!cqdma->pc[i]) 813 return -ENOMEM; 814 815 INIT_LIST_HEAD(&cqdma->pc[i]->queue); 816 spin_lock_init(&cqdma->pc[i]->lock); 817 refcount_set(&cqdma->pc[i]->refcnt, 0); 818 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i); 819 if (IS_ERR(cqdma->pc[i]->base)) 820 return PTR_ERR(cqdma->pc[i]->base); 821 822 /* allocate IRQ resource */ 823 err = platform_get_irq(pdev, i); 824 if (err < 0) 825 return err; 826 cqdma->pc[i]->irq = err; 827 828 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq, 829 mtk_cqdma_irq, 0, dev_name(&pdev->dev), 830 cqdma); 831 if (err) { 832 dev_err(&pdev->dev, 833 "request_irq failed with err %d\n", err); 834 return -EINVAL; 835 } 836 } 837 838 /* allocate resource for VCs */ 839 cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests, 840 sizeof(*cqdma->vc), GFP_KERNEL); 841 if (!cqdma->vc) 842 return -ENOMEM; 843 844 for (i = 0; i < cqdma->dma_requests; i++) { 845 vc = &cqdma->vc[i]; 846 vc->vc.desc_free = mtk_cqdma_vdesc_free; 847 vchan_init(&vc->vc, dd); 848 init_completion(&vc->issue_completion); 849 } 850 851 err = dma_async_device_register(dd); 852 if (err) 853 return err; 854 855 err = of_dma_controller_register(pdev->dev.of_node, 856 of_dma_xlate_by_chan_id, cqdma); 857 if (err) { 858 dev_err(&pdev->dev, 859 "MediaTek CQDMA OF registration failed %d\n", err); 860 goto err_unregister; 861 } 862 863 err = mtk_cqdma_hw_init(cqdma); 864 if (err) { 865 dev_err(&pdev->dev, 866 "MediaTek CQDMA HW initialization failed %d\n", err); 867 goto err_unregister; 868 } 869 870 platform_set_drvdata(pdev, cqdma); 871 872 /* initialize tasklet for each PC */ 873 for (i = 0; i < cqdma->dma_channels; ++i) 874 tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb); 875 876 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n"); 877 878 return 0; 879 880 err_unregister: 881 dma_async_device_unregister(dd); 882 883 return err; 884 } 885 886 static void mtk_cqdma_remove(struct platform_device *pdev) 887 { 888 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); 889 struct mtk_cqdma_vchan *vc; 890 unsigned long flags; 891 int i; 892 893 /* kill VC task */ 894 for (i = 0; i < cqdma->dma_requests; i++) { 895 vc = &cqdma->vc[i]; 896 897 list_del(&vc->vc.chan.device_node); 898 tasklet_kill(&vc->vc.task); 899 } 900 901 /* disable interrupt */ 902 for (i = 0; i < cqdma->dma_channels; i++) { 903 spin_lock_irqsave(&cqdma->pc[i]->lock, flags); 904 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN, 905 MTK_CQDMA_INT_EN_BIT); 906 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags); 907 908 /* Waits for any pending IRQ handlers to complete */ 909 synchronize_irq(cqdma->pc[i]->irq); 910 911 tasklet_kill(&cqdma->pc[i]->tasklet); 912 } 913 914 /* disable hardware */ 915 mtk_cqdma_hw_deinit(cqdma); 916 917 dma_async_device_unregister(&cqdma->ddev); 918 of_dma_controller_free(pdev->dev.of_node); 919 } 920 921 static struct platform_driver mtk_cqdma_driver = { 922 .probe = mtk_cqdma_probe, 923 .remove = mtk_cqdma_remove, 924 .driver = { 925 .name = KBUILD_MODNAME, 926 .of_match_table = mtk_cqdma_match, 927 }, 928 }; 929 module_platform_driver(mtk_cqdma_driver); 930 931 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver"); 932 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>"); 933 MODULE_LICENSE("GPL v2"); 934