1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 MediaTek Inc. 3 4 /* 5 * Driver for MediaTek High-Speed DMA Controller 6 * 7 * Author: Sean Wang <sean.wang@mediatek.com> 8 * 9 */ 10 11 #include <linux/bitops.h> 12 #include <linux/clk.h> 13 #include <linux/dmaengine.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/err.h> 16 #include <linux/iopoll.h> 17 #include <linux/list.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_device.h> 21 #include <linux/of_dma.h> 22 #include <linux/platform_device.h> 23 #include <linux/pm_runtime.h> 24 #include <linux/refcount.h> 25 #include <linux/slab.h> 26 27 #include "../virt-dma.h" 28 29 #define MTK_HSDMA_USEC_POLL 20 30 #define MTK_HSDMA_TIMEOUT_POLL 200000 31 #define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 32 33 /* The default number of virtual channel */ 34 #define MTK_HSDMA_NR_VCHANS 3 35 36 /* Only one physical channel supported */ 37 #define MTK_HSDMA_NR_MAX_PCHANS 1 38 39 /* Macro for physical descriptor (PD) manipulation */ 40 /* The number of PD which must be 2 of power */ 41 #define MTK_DMA_SIZE 64 42 #define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1)) 43 #define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1)) 44 #define MTK_HSDMA_MAX_LEN 0x3f80 45 #define MTK_HSDMA_ALIGN_SIZE 4 46 #define MTK_HSDMA_PLEN_MASK 0x3fff 47 #define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16) 48 #define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK) 49 50 /* Registers for underlying ring manipulation */ 51 #define MTK_HSDMA_TX_BASE 0x0 52 #define MTK_HSDMA_TX_CNT 0x4 53 #define MTK_HSDMA_TX_CPU 0x8 54 #define MTK_HSDMA_TX_DMA 0xc 55 #define MTK_HSDMA_RX_BASE 0x100 56 #define MTK_HSDMA_RX_CNT 0x104 57 #define MTK_HSDMA_RX_CPU 0x108 58 #define MTK_HSDMA_RX_DMA 0x10c 59 60 /* Registers for global setup */ 61 #define MTK_HSDMA_GLO 0x204 62 #define MTK_HSDMA_GLO_MULTI_DMA BIT(10) 63 #define MTK_HSDMA_TX_WB_DDONE BIT(6) 64 #define MTK_HSDMA_BURST_64BYTES (0x2 << 4) 65 #define MTK_HSDMA_GLO_RX_BUSY BIT(3) 66 #define MTK_HSDMA_GLO_RX_DMA BIT(2) 67 #define MTK_HSDMA_GLO_TX_BUSY BIT(1) 68 #define MTK_HSDMA_GLO_TX_DMA BIT(0) 69 #define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \ 70 MTK_HSDMA_GLO_RX_DMA) 71 #define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \ 72 MTK_HSDMA_GLO_TX_BUSY) 73 #define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \ 74 MTK_HSDMA_GLO_RX_DMA | \ 75 MTK_HSDMA_TX_WB_DDONE | \ 76 MTK_HSDMA_BURST_64BYTES | \ 77 MTK_HSDMA_GLO_MULTI_DMA) 78 79 /* Registers for reset */ 80 #define MTK_HSDMA_RESET 0x208 81 #define MTK_HSDMA_RST_TX BIT(0) 82 #define MTK_HSDMA_RST_RX BIT(16) 83 84 /* Registers for interrupt control */ 85 #define MTK_HSDMA_DLYINT 0x20c 86 #define MTK_HSDMA_RXDLY_INT_EN BIT(15) 87 88 /* Interrupt fires when the pending number's more than the specified */ 89 #define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8) 90 91 /* Interrupt fires when the pending time's more than the specified in 20 us */ 92 #define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f) 93 #define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \ 94 MTK_HSDMA_RXMAX_PINT(20) | \ 95 MTK_HSDMA_RXMAX_PTIME(20)) 96 #define MTK_HSDMA_INT_STATUS 0x220 97 #define MTK_HSDMA_INT_ENABLE 0x228 98 #define MTK_HSDMA_INT_RXDONE BIT(16) 99 100 enum mtk_hsdma_vdesc_flag { 101 MTK_HSDMA_VDESC_FINISHED = 0x01, 102 }; 103 104 #define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED) 105 106 /** 107 * struct mtk_hsdma_pdesc - This is the struct holding info describing physical 108 * descriptor (PD) and its placement must be kept at 109 * 4-bytes alignment in little endian order. 110 * @desc1: | The control pad used to indicate hardware how to 111 * @desc2: | deal with the descriptor such as source and 112 * @desc3: | destination address and data length. The maximum 113 * @desc4: | data length each pdesc can handle is 0x3f80 bytes 114 */ 115 struct mtk_hsdma_pdesc { 116 __le32 desc1; 117 __le32 desc2; 118 __le32 desc3; 119 __le32 desc4; 120 } __packed __aligned(4); 121 122 /** 123 * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual 124 * descriptor (VD) 125 * @vd: An instance for struct virt_dma_desc 126 * @len: The total data size device wants to move 127 * @residue: The remaining data size device will move 128 * @dest: The destination address device wants to move to 129 * @src: The source address device wants to move from 130 */ 131 struct mtk_hsdma_vdesc { 132 struct virt_dma_desc vd; 133 size_t len; 134 size_t residue; 135 dma_addr_t dest; 136 dma_addr_t src; 137 }; 138 139 /** 140 * struct mtk_hsdma_cb - This is the struct holding extra info required for RX 141 * ring to know what relevant VD the PD is being 142 * mapped to. 143 * @vd: Pointer to the relevant VD. 144 * @flag: Flag indicating what action should be taken when VD 145 * is completed. 146 */ 147 struct mtk_hsdma_cb { 148 struct virt_dma_desc *vd; 149 enum mtk_hsdma_vdesc_flag flag; 150 }; 151 152 /** 153 * struct mtk_hsdma_ring - This struct holds info describing underlying ring 154 * space 155 * @txd: The descriptor TX ring which describes DMA source 156 * information 157 * @rxd: The descriptor RX ring which describes DMA 158 * destination information 159 * @cb: The extra information pointed at by RX ring 160 * @tphys: The physical addr of TX ring 161 * @rphys: The physical addr of RX ring 162 * @cur_tptr: Pointer to the next free descriptor used by the host 163 * @cur_rptr: Pointer to the last done descriptor by the device 164 */ 165 struct mtk_hsdma_ring { 166 struct mtk_hsdma_pdesc *txd; 167 struct mtk_hsdma_pdesc *rxd; 168 struct mtk_hsdma_cb *cb; 169 dma_addr_t tphys; 170 dma_addr_t rphys; 171 u16 cur_tptr; 172 u16 cur_rptr; 173 }; 174 175 /** 176 * struct mtk_hsdma_pchan - This is the struct holding info describing physical 177 * channel (PC) 178 * @ring: An instance for the underlying ring 179 * @sz_ring: Total size allocated for the ring 180 * @nr_free: Total number of free rooms in the ring. It would 181 * be accessed and updated frequently between IRQ 182 * context and user context to reflect whether ring 183 * can accept requests from VD. 184 */ 185 struct mtk_hsdma_pchan { 186 struct mtk_hsdma_ring ring; 187 size_t sz_ring; 188 atomic_t nr_free; 189 }; 190 191 /** 192 * struct mtk_hsdma_vchan - This is the struct holding info describing virtual 193 * channel (VC) 194 * @vc: An instance for struct virt_dma_chan 195 * @issue_completion: The wait for all issued descriptors completited 196 * @issue_synchronize: Bool indicating channel synchronization starts 197 * @desc_hw_processing: List those descriptors the hardware is processing, 198 * which is protected by vc.lock 199 */ 200 struct mtk_hsdma_vchan { 201 struct virt_dma_chan vc; 202 struct completion issue_completion; 203 bool issue_synchronize; 204 struct list_head desc_hw_processing; 205 }; 206 207 /** 208 * struct mtk_hsdma_soc - This is the struct holding differences among SoCs 209 * @ddone: Bit mask for DDONE 210 * @ls0: Bit mask for LS0 211 */ 212 struct mtk_hsdma_soc { 213 __le32 ddone; 214 __le32 ls0; 215 }; 216 217 /** 218 * struct mtk_hsdma_device - This is the struct holding info describing HSDMA 219 * device 220 * @ddev: An instance for struct dma_device 221 * @base: The mapped register I/O base 222 * @clk: The clock that device internal is using 223 * @irq: The IRQ that device are using 224 * @dma_requests: The number of VCs the device supports to 225 * @vc: The pointer to all available VCs 226 * @pc: The pointer to the underlying PC 227 * @pc_refcnt: Track how many VCs are using the PC 228 * @lock: Lock protect agaisting multiple VCs access PC 229 * @soc: The pointer to area holding differences among 230 * vaious platform 231 */ 232 struct mtk_hsdma_device { 233 struct dma_device ddev; 234 void __iomem *base; 235 struct clk *clk; 236 u32 irq; 237 238 u32 dma_requests; 239 struct mtk_hsdma_vchan *vc; 240 struct mtk_hsdma_pchan *pc; 241 refcount_t pc_refcnt; 242 243 /* Lock used to protect against multiple VCs access PC */ 244 spinlock_t lock; 245 246 const struct mtk_hsdma_soc *soc; 247 }; 248 249 static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan) 250 { 251 return container_of(chan->device, struct mtk_hsdma_device, ddev); 252 } 253 254 static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan) 255 { 256 return container_of(chan, struct mtk_hsdma_vchan, vc.chan); 257 } 258 259 static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd) 260 { 261 return container_of(vd, struct mtk_hsdma_vdesc, vd); 262 } 263 264 static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma) 265 { 266 return hsdma->ddev.dev; 267 } 268 269 static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg) 270 { 271 return readl(hsdma->base + reg); 272 } 273 274 static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) 275 { 276 writel(val, hsdma->base + reg); 277 } 278 279 static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg, 280 u32 mask, u32 set) 281 { 282 u32 val; 283 284 val = mtk_dma_read(hsdma, reg); 285 val &= ~mask; 286 val |= set; 287 mtk_dma_write(hsdma, reg, val); 288 } 289 290 static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) 291 { 292 mtk_dma_rmw(hsdma, reg, 0, val); 293 } 294 295 static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) 296 { 297 mtk_dma_rmw(hsdma, reg, val, 0); 298 } 299 300 static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd) 301 { 302 kfree(container_of(vd, struct mtk_hsdma_vdesc, vd)); 303 } 304 305 static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma) 306 { 307 u32 status = 0; 308 309 return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status, 310 !(status & MTK_HSDMA_GLO_BUSY), 311 MTK_HSDMA_USEC_POLL, 312 MTK_HSDMA_TIMEOUT_POLL); 313 } 314 315 static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma, 316 struct mtk_hsdma_pchan *pc) 317 { 318 struct mtk_hsdma_ring *ring = &pc->ring; 319 int err; 320 321 memset(pc, 0, sizeof(*pc)); 322 323 /* 324 * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring 325 * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring. 326 */ 327 pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd); 328 ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring, 329 &ring->tphys, GFP_NOWAIT); 330 if (!ring->txd) 331 return -ENOMEM; 332 333 ring->rxd = &ring->txd[MTK_DMA_SIZE]; 334 ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd); 335 ring->cur_tptr = 0; 336 ring->cur_rptr = MTK_DMA_SIZE - 1; 337 338 ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT); 339 if (!ring->cb) { 340 err = -ENOMEM; 341 goto err_free_dma; 342 } 343 344 atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1); 345 346 /* Disable HSDMA and wait for the completion */ 347 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); 348 err = mtk_hsdma_busy_wait(hsdma); 349 if (err) 350 goto err_free_cb; 351 352 /* Reset */ 353 mtk_dma_set(hsdma, MTK_HSDMA_RESET, 354 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); 355 mtk_dma_clr(hsdma, MTK_HSDMA_RESET, 356 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); 357 358 /* Setup HSDMA initial pointer in the ring */ 359 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys); 360 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE); 361 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr); 362 mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0); 363 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys); 364 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE); 365 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr); 366 mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0); 367 368 /* Enable HSDMA */ 369 mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); 370 371 /* Setup delayed interrupt */ 372 mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT); 373 374 /* Enable interrupt */ 375 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 376 377 return 0; 378 379 err_free_cb: 380 kfree(ring->cb); 381 382 err_free_dma: 383 dma_free_coherent(hsdma2dev(hsdma), 384 pc->sz_ring, ring->txd, ring->tphys); 385 return err; 386 } 387 388 static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma, 389 struct mtk_hsdma_pchan *pc) 390 { 391 struct mtk_hsdma_ring *ring = &pc->ring; 392 393 /* Disable HSDMA and then wait for the completion */ 394 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); 395 mtk_hsdma_busy_wait(hsdma); 396 397 /* Reset pointer in the ring */ 398 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 399 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0); 400 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0); 401 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0); 402 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0); 403 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0); 404 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1); 405 406 kfree(ring->cb); 407 408 dma_free_coherent(hsdma2dev(hsdma), 409 pc->sz_ring, ring->txd, ring->tphys); 410 } 411 412 static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma, 413 struct mtk_hsdma_pchan *pc, 414 struct mtk_hsdma_vdesc *hvd) 415 { 416 struct mtk_hsdma_ring *ring = &pc->ring; 417 struct mtk_hsdma_pdesc *txd, *rxd; 418 u16 reserved, prev, tlen, num_sgs; 419 unsigned long flags; 420 421 /* Protect against PC is accessed by multiple VCs simultaneously */ 422 spin_lock_irqsave(&hsdma->lock, flags); 423 424 /* 425 * Reserve rooms, where pc->nr_free is used to track how many free 426 * rooms in the ring being updated in user and IRQ context. 427 */ 428 num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN); 429 reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free)); 430 431 if (!reserved) { 432 spin_unlock_irqrestore(&hsdma->lock, flags); 433 return -ENOSPC; 434 } 435 436 atomic_sub(reserved, &pc->nr_free); 437 438 while (reserved--) { 439 /* Limit size by PD capability for valid data moving */ 440 tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ? 441 MTK_HSDMA_MAX_LEN : hvd->len; 442 443 /* 444 * Setup PDs using the remaining VD info mapped on those 445 * reserved rooms. And since RXD is shared memory between the 446 * host and the device allocated by dma_alloc_coherent call, 447 * the helper macro WRITE_ONCE can ensure the data written to 448 * RAM would really happens. 449 */ 450 txd = &ring->txd[ring->cur_tptr]; 451 WRITE_ONCE(txd->desc1, hvd->src); 452 WRITE_ONCE(txd->desc2, 453 hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen)); 454 455 rxd = &ring->rxd[ring->cur_tptr]; 456 WRITE_ONCE(rxd->desc1, hvd->dest); 457 WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen)); 458 459 /* Associate VD, the PD belonged to */ 460 ring->cb[ring->cur_tptr].vd = &hvd->vd; 461 462 /* Move forward the pointer of TX ring */ 463 ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr, 464 MTK_DMA_SIZE); 465 466 /* Update VD with remaining data */ 467 hvd->src += tlen; 468 hvd->dest += tlen; 469 hvd->len -= tlen; 470 } 471 472 /* 473 * Tagging flag for the last PD for VD will be responsible for 474 * completing VD. 475 */ 476 if (!hvd->len) { 477 prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE); 478 ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED; 479 } 480 481 /* Ensure all changes indeed done before we're going on */ 482 wmb(); 483 484 /* 485 * Updating into hardware the pointer of TX ring lets HSDMA to take 486 * action for those pending PDs. 487 */ 488 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr); 489 490 spin_unlock_irqrestore(&hsdma->lock, flags); 491 492 return 0; 493 } 494 495 static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma, 496 struct mtk_hsdma_vchan *hvc) 497 { 498 struct virt_dma_desc *vd, *vd2; 499 int err; 500 501 lockdep_assert_held(&hvc->vc.lock); 502 503 list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) { 504 struct mtk_hsdma_vdesc *hvd; 505 506 hvd = to_hsdma_vdesc(vd); 507 508 /* Map VD into PC and all VCs shares a single PC */ 509 err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd); 510 511 /* 512 * Move VD from desc_issued to desc_hw_processing when entire 513 * VD is fit into available PDs. Otherwise, the uncompleted 514 * VDs would stay in list desc_issued and then restart the 515 * processing as soon as possible once underlying ring space 516 * got freed. 517 */ 518 if (err == -ENOSPC || hvd->len > 0) 519 break; 520 521 /* 522 * The extra list desc_hw_processing is used because 523 * hardware can't provide sufficient information allowing us 524 * to know what VDs are still working on the underlying ring. 525 * Through the additional list, it can help us to implement 526 * terminate_all, residue calculation and such thing needed 527 * to know detail descriptor status on the hardware. 528 */ 529 list_move_tail(&vd->node, &hvc->desc_hw_processing); 530 } 531 } 532 533 static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma) 534 { 535 struct mtk_hsdma_vchan *hvc; 536 struct mtk_hsdma_pdesc *rxd; 537 struct mtk_hsdma_vdesc *hvd; 538 struct mtk_hsdma_pchan *pc; 539 struct mtk_hsdma_cb *cb; 540 int i = MTK_DMA_SIZE; 541 __le32 desc2; 542 u32 status; 543 u16 next; 544 545 /* Read IRQ status */ 546 status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS); 547 if (unlikely(!(status & MTK_HSDMA_INT_RXDONE))) 548 goto rx_done; 549 550 pc = hsdma->pc; 551 552 /* 553 * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to 554 * reclaim these finished descriptors: The most number of PDs the ISR 555 * can handle at one time shouldn't be more than MTK_DMA_SIZE so we 556 * take it as limited count instead of just using a dangerous infinite 557 * poll. 558 */ 559 while (i--) { 560 next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr, 561 MTK_DMA_SIZE); 562 rxd = &pc->ring.rxd[next]; 563 564 /* 565 * If MTK_HSDMA_DESC_DDONE is no specified, that means data 566 * moving for the PD is still under going. 567 */ 568 desc2 = READ_ONCE(rxd->desc2); 569 if (!(desc2 & hsdma->soc->ddone)) 570 break; 571 572 cb = &pc->ring.cb[next]; 573 if (unlikely(!cb->vd)) { 574 dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n"); 575 break; 576 } 577 578 /* Update residue of VD the associated PD belonged to */ 579 hvd = to_hsdma_vdesc(cb->vd); 580 hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2); 581 582 /* Complete VD until the relevant last PD is finished */ 583 if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) { 584 hvc = to_hsdma_vchan(cb->vd->tx.chan); 585 586 spin_lock(&hvc->vc.lock); 587 588 /* Remove VD from list desc_hw_processing */ 589 list_del(&cb->vd->node); 590 591 /* Add VD into list desc_completed */ 592 vchan_cookie_complete(cb->vd); 593 594 if (hvc->issue_synchronize && 595 list_empty(&hvc->desc_hw_processing)) { 596 complete(&hvc->issue_completion); 597 hvc->issue_synchronize = false; 598 } 599 spin_unlock(&hvc->vc.lock); 600 601 cb->flag = 0; 602 } 603 604 cb->vd = NULL; 605 606 /* 607 * Recycle the RXD with the helper WRITE_ONCE that can ensure 608 * data written into RAM would really happens. 609 */ 610 WRITE_ONCE(rxd->desc1, 0); 611 WRITE_ONCE(rxd->desc2, 0); 612 pc->ring.cur_rptr = next; 613 614 /* Release rooms */ 615 atomic_inc(&pc->nr_free); 616 } 617 618 /* Ensure all changes indeed done before we're going on */ 619 wmb(); 620 621 /* Update CPU pointer for those completed PDs */ 622 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr); 623 624 /* 625 * Acking the pending IRQ allows hardware no longer to keep the used 626 * IRQ line in certain trigger state when software has completed all 627 * the finished physical descriptors. 628 */ 629 if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1) 630 mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status); 631 632 /* ASAP handles pending VDs in all VCs after freeing some rooms */ 633 for (i = 0; i < hsdma->dma_requests; i++) { 634 hvc = &hsdma->vc[i]; 635 spin_lock(&hvc->vc.lock); 636 mtk_hsdma_issue_vchan_pending(hsdma, hvc); 637 spin_unlock(&hvc->vc.lock); 638 } 639 640 rx_done: 641 /* All completed PDs are cleaned up, so enable interrupt again */ 642 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 643 } 644 645 static irqreturn_t mtk_hsdma_irq(int irq, void *devid) 646 { 647 struct mtk_hsdma_device *hsdma = devid; 648 649 /* 650 * Disable interrupt until all completed PDs are cleaned up in 651 * mtk_hsdma_free_rooms call. 652 */ 653 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); 654 655 mtk_hsdma_free_rooms_in_ring(hsdma); 656 657 return IRQ_HANDLED; 658 } 659 660 static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c, 661 dma_cookie_t cookie) 662 { 663 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 664 struct virt_dma_desc *vd; 665 666 list_for_each_entry(vd, &hvc->desc_hw_processing, node) 667 if (vd->tx.cookie == cookie) 668 return vd; 669 670 list_for_each_entry(vd, &hvc->vc.desc_issued, node) 671 if (vd->tx.cookie == cookie) 672 return vd; 673 674 return NULL; 675 } 676 677 static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c, 678 dma_cookie_t cookie, 679 struct dma_tx_state *txstate) 680 { 681 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 682 struct mtk_hsdma_vdesc *hvd; 683 struct virt_dma_desc *vd; 684 enum dma_status ret; 685 unsigned long flags; 686 size_t bytes = 0; 687 688 ret = dma_cookie_status(c, cookie, txstate); 689 if (ret == DMA_COMPLETE || !txstate) 690 return ret; 691 692 spin_lock_irqsave(&hvc->vc.lock, flags); 693 vd = mtk_hsdma_find_active_desc(c, cookie); 694 spin_unlock_irqrestore(&hvc->vc.lock, flags); 695 696 if (vd) { 697 hvd = to_hsdma_vdesc(vd); 698 bytes = hvd->residue; 699 } 700 701 dma_set_residue(txstate, bytes); 702 703 return ret; 704 } 705 706 static void mtk_hsdma_issue_pending(struct dma_chan *c) 707 { 708 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); 709 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 710 unsigned long flags; 711 712 spin_lock_irqsave(&hvc->vc.lock, flags); 713 714 if (vchan_issue_pending(&hvc->vc)) 715 mtk_hsdma_issue_vchan_pending(hsdma, hvc); 716 717 spin_unlock_irqrestore(&hvc->vc.lock, flags); 718 } 719 720 static struct dma_async_tx_descriptor * 721 mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, 722 dma_addr_t src, size_t len, unsigned long flags) 723 { 724 struct mtk_hsdma_vdesc *hvd; 725 726 hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT); 727 if (!hvd) 728 return NULL; 729 730 hvd->len = len; 731 hvd->residue = len; 732 hvd->src = src; 733 hvd->dest = dest; 734 735 return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags); 736 } 737 738 static int mtk_hsdma_free_inactive_desc(struct dma_chan *c) 739 { 740 struct virt_dma_chan *vc = to_virt_chan(c); 741 unsigned long flags; 742 LIST_HEAD(head); 743 744 spin_lock_irqsave(&vc->lock, flags); 745 list_splice_tail_init(&vc->desc_allocated, &head); 746 list_splice_tail_init(&vc->desc_submitted, &head); 747 list_splice_tail_init(&vc->desc_issued, &head); 748 spin_unlock_irqrestore(&vc->lock, flags); 749 750 /* At the point, we don't expect users put descriptor into VC again */ 751 vchan_dma_desc_free_list(vc, &head); 752 753 return 0; 754 } 755 756 static void mtk_hsdma_free_active_desc(struct dma_chan *c) 757 { 758 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c); 759 bool sync_needed = false; 760 761 /* 762 * Once issue_synchronize is being set, which means once the hardware 763 * consumes all descriptors for the channel in the ring, the 764 * synchronization must be notified immediately it is completed. 765 */ 766 spin_lock(&hvc->vc.lock); 767 if (!list_empty(&hvc->desc_hw_processing)) { 768 hvc->issue_synchronize = true; 769 sync_needed = true; 770 } 771 spin_unlock(&hvc->vc.lock); 772 773 if (sync_needed) 774 wait_for_completion(&hvc->issue_completion); 775 /* 776 * At the point, we expect that all remaining descriptors in the ring 777 * for the channel should be all processing done. 778 */ 779 WARN_ONCE(!list_empty(&hvc->desc_hw_processing), 780 "Desc pending still in list desc_hw_processing\n"); 781 782 /* Free all descriptors in list desc_completed */ 783 vchan_synchronize(&hvc->vc); 784 785 WARN_ONCE(!list_empty(&hvc->vc.desc_completed), 786 "Desc pending still in list desc_completed\n"); 787 } 788 789 static int mtk_hsdma_terminate_all(struct dma_chan *c) 790 { 791 /* 792 * Free pending descriptors not processed yet by hardware that have 793 * previously been submitted to the channel. 794 */ 795 mtk_hsdma_free_inactive_desc(c); 796 797 /* 798 * However, the DMA engine doesn't provide any way to stop these 799 * descriptors being processed currently by hardware. The only way is 800 * to just waiting until these descriptors are all processed completely 801 * through mtk_hsdma_free_active_desc call. 802 */ 803 mtk_hsdma_free_active_desc(c); 804 805 return 0; 806 } 807 808 static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c) 809 { 810 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); 811 int err; 812 813 /* 814 * Since HSDMA has only one PC, the resource for PC is being allocated 815 * when the first VC is being created and the other VCs would run on 816 * the same PC. 817 */ 818 if (!refcount_read(&hsdma->pc_refcnt)) { 819 err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc); 820 if (err) 821 return err; 822 /* 823 * refcount_inc would complain increment on 0; use-after-free. 824 * Thus, we need to explicitly set it as 1 initially. 825 */ 826 refcount_set(&hsdma->pc_refcnt, 1); 827 } else { 828 refcount_inc(&hsdma->pc_refcnt); 829 } 830 831 return 0; 832 } 833 834 static void mtk_hsdma_free_chan_resources(struct dma_chan *c) 835 { 836 struct mtk_hsdma_device *hsdma = to_hsdma_dev(c); 837 838 /* Free all descriptors in all lists on the VC */ 839 mtk_hsdma_terminate_all(c); 840 841 /* The resource for PC is not freed until all the VCs are destroyed */ 842 if (!refcount_dec_and_test(&hsdma->pc_refcnt)) 843 return; 844 845 mtk_hsdma_free_pchan(hsdma, hsdma->pc); 846 } 847 848 static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma) 849 { 850 int err; 851 852 pm_runtime_enable(hsdma2dev(hsdma)); 853 pm_runtime_get_sync(hsdma2dev(hsdma)); 854 855 err = clk_prepare_enable(hsdma->clk); 856 if (err) 857 return err; 858 859 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); 860 mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT); 861 862 return 0; 863 } 864 865 static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma) 866 { 867 mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0); 868 869 clk_disable_unprepare(hsdma->clk); 870 871 pm_runtime_put_sync(hsdma2dev(hsdma)); 872 pm_runtime_disable(hsdma2dev(hsdma)); 873 874 return 0; 875 } 876 877 static const struct mtk_hsdma_soc mt7623_soc = { 878 .ddone = BIT(31), 879 .ls0 = BIT(30), 880 }; 881 882 static const struct mtk_hsdma_soc mt7622_soc = { 883 .ddone = BIT(15), 884 .ls0 = BIT(14), 885 }; 886 887 static const struct of_device_id mtk_hsdma_match[] = { 888 { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc}, 889 { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc}, 890 { /* sentinel */ } 891 }; 892 MODULE_DEVICE_TABLE(of, mtk_hsdma_match); 893 894 static int mtk_hsdma_probe(struct platform_device *pdev) 895 { 896 struct mtk_hsdma_device *hsdma; 897 struct mtk_hsdma_vchan *vc; 898 struct dma_device *dd; 899 int i, err; 900 901 hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL); 902 if (!hsdma) 903 return -ENOMEM; 904 905 dd = &hsdma->ddev; 906 907 hsdma->base = devm_platform_ioremap_resource(pdev, 0); 908 if (IS_ERR(hsdma->base)) 909 return PTR_ERR(hsdma->base); 910 911 hsdma->soc = of_device_get_match_data(&pdev->dev); 912 if (!hsdma->soc) { 913 dev_err(&pdev->dev, "No device match found\n"); 914 return -ENODEV; 915 } 916 917 hsdma->clk = devm_clk_get(&pdev->dev, "hsdma"); 918 if (IS_ERR(hsdma->clk)) { 919 dev_err(&pdev->dev, "No clock for %s\n", 920 dev_name(&pdev->dev)); 921 return PTR_ERR(hsdma->clk); 922 } 923 924 err = platform_get_irq(pdev, 0); 925 if (err < 0) 926 return err; 927 hsdma->irq = err; 928 929 refcount_set(&hsdma->pc_refcnt, 0); 930 spin_lock_init(&hsdma->lock); 931 932 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 933 934 dd->copy_align = MTK_HSDMA_ALIGN_SIZE; 935 dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources; 936 dd->device_free_chan_resources = mtk_hsdma_free_chan_resources; 937 dd->device_tx_status = mtk_hsdma_tx_status; 938 dd->device_issue_pending = mtk_hsdma_issue_pending; 939 dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy; 940 dd->device_terminate_all = mtk_hsdma_terminate_all; 941 dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; 942 dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; 943 dd->directions = BIT(DMA_MEM_TO_MEM); 944 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 945 dd->dev = &pdev->dev; 946 INIT_LIST_HEAD(&dd->channels); 947 948 hsdma->dma_requests = MTK_HSDMA_NR_VCHANS; 949 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 950 "dma-requests", 951 &hsdma->dma_requests)) { 952 dev_info(&pdev->dev, 953 "Using %u as missing dma-requests property\n", 954 MTK_HSDMA_NR_VCHANS); 955 } 956 957 hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS, 958 sizeof(*hsdma->pc), GFP_KERNEL); 959 if (!hsdma->pc) 960 return -ENOMEM; 961 962 hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests, 963 sizeof(*hsdma->vc), GFP_KERNEL); 964 if (!hsdma->vc) 965 return -ENOMEM; 966 967 for (i = 0; i < hsdma->dma_requests; i++) { 968 vc = &hsdma->vc[i]; 969 vc->vc.desc_free = mtk_hsdma_vdesc_free; 970 vchan_init(&vc->vc, dd); 971 init_completion(&vc->issue_completion); 972 INIT_LIST_HEAD(&vc->desc_hw_processing); 973 } 974 975 err = dma_async_device_register(dd); 976 if (err) 977 return err; 978 979 err = of_dma_controller_register(pdev->dev.of_node, 980 of_dma_xlate_by_chan_id, hsdma); 981 if (err) { 982 dev_err(&pdev->dev, 983 "MediaTek HSDMA OF registration failed %d\n", err); 984 goto err_unregister; 985 } 986 987 mtk_hsdma_hw_init(hsdma); 988 989 err = devm_request_irq(&pdev->dev, hsdma->irq, 990 mtk_hsdma_irq, 0, 991 dev_name(&pdev->dev), hsdma); 992 if (err) { 993 dev_err(&pdev->dev, 994 "request_irq failed with err %d\n", err); 995 goto err_free; 996 } 997 998 platform_set_drvdata(pdev, hsdma); 999 1000 dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n"); 1001 1002 return 0; 1003 1004 err_free: 1005 mtk_hsdma_hw_deinit(hsdma); 1006 of_dma_controller_free(pdev->dev.of_node); 1007 err_unregister: 1008 dma_async_device_unregister(dd); 1009 1010 return err; 1011 } 1012 1013 static int mtk_hsdma_remove(struct platform_device *pdev) 1014 { 1015 struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev); 1016 struct mtk_hsdma_vchan *vc; 1017 int i; 1018 1019 /* Kill VC task */ 1020 for (i = 0; i < hsdma->dma_requests; i++) { 1021 vc = &hsdma->vc[i]; 1022 1023 list_del(&vc->vc.chan.device_node); 1024 tasklet_kill(&vc->vc.task); 1025 } 1026 1027 /* Disable DMA interrupt */ 1028 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0); 1029 1030 /* Waits for any pending IRQ handlers to complete */ 1031 synchronize_irq(hsdma->irq); 1032 1033 /* Disable hardware */ 1034 mtk_hsdma_hw_deinit(hsdma); 1035 1036 dma_async_device_unregister(&hsdma->ddev); 1037 of_dma_controller_free(pdev->dev.of_node); 1038 1039 return 0; 1040 } 1041 1042 static struct platform_driver mtk_hsdma_driver = { 1043 .probe = mtk_hsdma_probe, 1044 .remove = mtk_hsdma_remove, 1045 .driver = { 1046 .name = KBUILD_MODNAME, 1047 .of_match_table = mtk_hsdma_match, 1048 }, 1049 }; 1050 module_platform_driver(mtk_hsdma_driver); 1051 1052 MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver"); 1053 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>"); 1054 MODULE_LICENSE("GPL v2"); 1055