1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2018-2019 MediaTek Inc.
3
4 /*
5 * Driver for MediaTek Command-Queue DMA Controller
6 *
7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
8 *
9 */
10
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/iopoll.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_dma.h>
22 #include <linux/platform_device.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/refcount.h>
25 #include <linux/slab.h>
26
27 #include "../virt-dma.h"
28
29 #define MTK_CQDMA_USEC_POLL 10
30 #define MTK_CQDMA_TIMEOUT_POLL 1000
31 #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
32 #define MTK_CQDMA_ALIGN_SIZE 1
33
34 /* The default number of virtual channel */
35 #define MTK_CQDMA_NR_VCHANS 32
36
37 /* The default number of physical channel */
38 #define MTK_CQDMA_NR_PCHANS 3
39
40 /* Registers for underlying dma manipulation */
41 #define MTK_CQDMA_INT_FLAG 0x0
42 #define MTK_CQDMA_INT_EN 0x4
43 #define MTK_CQDMA_EN 0x8
44 #define MTK_CQDMA_RESET 0xc
45 #define MTK_CQDMA_FLUSH 0x14
46 #define MTK_CQDMA_SRC 0x1c
47 #define MTK_CQDMA_DST 0x20
48 #define MTK_CQDMA_LEN1 0x24
49 #define MTK_CQDMA_LEN2 0x28
50 #define MTK_CQDMA_SRC2 0x60
51 #define MTK_CQDMA_DST2 0x64
52
53 /* Registers setting */
54 #define MTK_CQDMA_EN_BIT BIT(0)
55 #define MTK_CQDMA_INT_FLAG_BIT BIT(0)
56 #define MTK_CQDMA_INT_EN_BIT BIT(0)
57 #define MTK_CQDMA_FLUSH_BIT BIT(0)
58
59 #define MTK_CQDMA_WARM_RST_BIT BIT(0)
60 #define MTK_CQDMA_HARD_RST_BIT BIT(1)
61
62 #define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
63 #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
64 #define MTK_CQDMA_ADDR2_SHFIT (32)
65
66 /**
67 * struct mtk_cqdma_vdesc - The struct holding info describing virtual
68 * descriptor (CVD)
69 * @vd: An instance for struct virt_dma_desc
70 * @len: The total data size device wants to move
71 * @residue: The remaining data size device will move
72 * @dest: The destination address device wants to move to
73 * @src: The source address device wants to move from
74 * @ch: The pointer to the corresponding dma channel
75 * @node: The lise_head struct to build link-list for VDs
76 * @parent: The pointer to the parent CVD
77 */
78 struct mtk_cqdma_vdesc {
79 struct virt_dma_desc vd;
80 size_t len;
81 size_t residue;
82 dma_addr_t dest;
83 dma_addr_t src;
84 struct dma_chan *ch;
85
86 struct list_head node;
87 struct mtk_cqdma_vdesc *parent;
88 };
89
90 /**
91 * struct mtk_cqdma_pchan - The struct holding info describing physical
92 * channel (PC)
93 * @queue: Queue for the PDs issued to this PC
94 * @base: The mapped register I/O base of this PC
95 * @irq: The IRQ that this PC are using
96 * @refcnt: Track how many VCs are using this PC
97 * @tasklet: Tasklet for this PC
98 * @lock: Lock protect agaisting multiple VCs access PC
99 */
100 struct mtk_cqdma_pchan {
101 struct list_head queue;
102 void __iomem *base;
103 u32 irq;
104
105 refcount_t refcnt;
106
107 struct tasklet_struct tasklet;
108
109 /* lock to protect PC */
110 spinlock_t lock;
111 };
112
113 /**
114 * struct mtk_cqdma_vchan - The struct holding info describing virtual
115 * channel (VC)
116 * @vc: An instance for struct virt_dma_chan
117 * @pc: The pointer to the underlying PC
118 * @issue_completion: The wait for all issued descriptors completited
119 * @issue_synchronize: Bool indicating channel synchronization starts
120 */
121 struct mtk_cqdma_vchan {
122 struct virt_dma_chan vc;
123 struct mtk_cqdma_pchan *pc;
124 struct completion issue_completion;
125 bool issue_synchronize;
126 };
127
128 /**
129 * struct mtk_cqdma_device - The struct holding info describing CQDMA
130 * device
131 * @ddev: An instance for struct dma_device
132 * @clk: The clock that device internal is using
133 * @dma_requests: The number of VCs the device supports to
134 * @dma_channels: The number of PCs the device supports to
135 * @vc: The pointer to all available VCs
136 * @pc: The pointer to all the underlying PCs
137 */
138 struct mtk_cqdma_device {
139 struct dma_device ddev;
140 struct clk *clk;
141
142 u32 dma_requests;
143 u32 dma_channels;
144 struct mtk_cqdma_vchan *vc;
145 struct mtk_cqdma_pchan **pc;
146 };
147
to_cqdma_dev(struct dma_chan * chan)148 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
149 {
150 return container_of(chan->device, struct mtk_cqdma_device, ddev);
151 }
152
to_cqdma_vchan(struct dma_chan * chan)153 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
154 {
155 return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
156 }
157
to_cqdma_vdesc(struct virt_dma_desc * vd)158 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
159 {
160 return container_of(vd, struct mtk_cqdma_vdesc, vd);
161 }
162
cqdma2dev(struct mtk_cqdma_device * cqdma)163 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
164 {
165 return cqdma->ddev.dev;
166 }
167
mtk_dma_read(struct mtk_cqdma_pchan * pc,u32 reg)168 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
169 {
170 return readl(pc->base + reg);
171 }
172
mtk_dma_write(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)173 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
174 {
175 writel_relaxed(val, pc->base + reg);
176 }
177
mtk_dma_rmw(struct mtk_cqdma_pchan * pc,u32 reg,u32 mask,u32 set)178 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
179 u32 mask, u32 set)
180 {
181 u32 val;
182
183 val = mtk_dma_read(pc, reg);
184 val &= ~mask;
185 val |= set;
186 mtk_dma_write(pc, reg, val);
187 }
188
mtk_dma_set(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)189 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
190 {
191 mtk_dma_rmw(pc, reg, 0, val);
192 }
193
mtk_dma_clr(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)194 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
195 {
196 mtk_dma_rmw(pc, reg, val, 0);
197 }
198
mtk_cqdma_vdesc_free(struct virt_dma_desc * vd)199 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
200 {
201 kfree(to_cqdma_vdesc(vd));
202 }
203
mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan * pc,bool atomic)204 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
205 {
206 u32 status = 0;
207
208 if (!atomic)
209 return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
210 status,
211 !(status & MTK_CQDMA_EN_BIT),
212 MTK_CQDMA_USEC_POLL,
213 MTK_CQDMA_TIMEOUT_POLL);
214
215 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
216 status,
217 !(status & MTK_CQDMA_EN_BIT),
218 MTK_CQDMA_USEC_POLL,
219 MTK_CQDMA_TIMEOUT_POLL);
220 }
221
mtk_cqdma_hard_reset(struct mtk_cqdma_pchan * pc)222 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
223 {
224 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
225 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226
227 return mtk_cqdma_poll_engine_done(pc, true);
228 }
229
mtk_cqdma_start(struct mtk_cqdma_pchan * pc,struct mtk_cqdma_vdesc * cvd)230 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
231 struct mtk_cqdma_vdesc *cvd)
232 {
233 /* wait for the previous transaction done */
234 if (mtk_cqdma_poll_engine_done(pc, true) < 0)
235 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
236
237 /* warm reset the dma engine for the new transaction */
238 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
239 if (mtk_cqdma_poll_engine_done(pc, true) < 0)
240 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
241
242 /* setup the source */
243 mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
244 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
245 mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
246 #else
247 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
248 #endif
249
250 /* setup the destination */
251 mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
252 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
253 mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
254 #else
255 mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
256 #endif
257
258 /* setup the length */
259 mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
260
261 /* start dma engine */
262 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
263 }
264
mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan * cvc)265 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
266 {
267 struct virt_dma_desc *vd, *vd2;
268 struct mtk_cqdma_pchan *pc = cvc->pc;
269 struct mtk_cqdma_vdesc *cvd;
270 bool trigger_engine = false;
271
272 lockdep_assert_held(&cvc->vc.lock);
273 lockdep_assert_held(&pc->lock);
274
275 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
276 /* need to trigger dma engine if PC's queue is empty */
277 if (list_empty(&pc->queue))
278 trigger_engine = true;
279
280 cvd = to_cqdma_vdesc(vd);
281
282 /* add VD into PC's queue */
283 list_add_tail(&cvd->node, &pc->queue);
284
285 /* start the dma engine */
286 if (trigger_engine)
287 mtk_cqdma_start(pc, cvd);
288
289 /* remove VD from list desc_issued */
290 list_del(&vd->node);
291 }
292 }
293
294 /*
295 * return true if this VC is active,
296 * meaning that there are VDs under processing by the PC
297 */
mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan * cvc)298 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
299 {
300 struct mtk_cqdma_vdesc *cvd;
301
302 list_for_each_entry(cvd, &cvc->pc->queue, node)
303 if (cvc == to_cqdma_vchan(cvd->ch))
304 return true;
305
306 return false;
307 }
308
309 /*
310 * return the pointer of the CVD that is just consumed by the PC
311 */
312 static struct mtk_cqdma_vdesc
mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan * pc)313 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
314 {
315 struct mtk_cqdma_vchan *cvc;
316 struct mtk_cqdma_vdesc *cvd, *ret = NULL;
317
318 /* consume a CVD from PC's queue */
319 cvd = list_first_entry_or_null(&pc->queue,
320 struct mtk_cqdma_vdesc, node);
321 if (unlikely(!cvd || !cvd->parent))
322 return NULL;
323
324 cvc = to_cqdma_vchan(cvd->ch);
325 ret = cvd;
326
327 /* update residue of the parent CVD */
328 cvd->parent->residue -= cvd->len;
329
330 /* delete CVD from PC's queue */
331 list_del(&cvd->node);
332
333 spin_lock(&cvc->vc.lock);
334
335 /* check whether all the child CVDs completed */
336 if (!cvd->parent->residue) {
337 /* add the parent VD into list desc_completed */
338 vchan_cookie_complete(&cvd->parent->vd);
339
340 /* setup completion if this VC is under synchronization */
341 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
342 complete(&cvc->issue_completion);
343 cvc->issue_synchronize = false;
344 }
345 }
346
347 spin_unlock(&cvc->vc.lock);
348
349 /* start transaction for next CVD in the queue */
350 cvd = list_first_entry_or_null(&pc->queue,
351 struct mtk_cqdma_vdesc, node);
352 if (cvd)
353 mtk_cqdma_start(pc, cvd);
354
355 return ret;
356 }
357
mtk_cqdma_tasklet_cb(struct tasklet_struct * t)358 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
359 {
360 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
361 struct mtk_cqdma_vdesc *cvd = NULL;
362 unsigned long flags;
363
364 spin_lock_irqsave(&pc->lock, flags);
365 /* consume the queue */
366 cvd = mtk_cqdma_consume_work_queue(pc);
367 spin_unlock_irqrestore(&pc->lock, flags);
368
369 /* submit the next CVD */
370 if (cvd) {
371 dma_run_dependencies(&cvd->vd.tx);
372
373 /*
374 * free child CVD after completion.
375 * the parent CVD would be freed with desc_free by user.
376 */
377 if (cvd->parent != cvd)
378 kfree(cvd);
379 }
380
381 /* re-enable interrupt before leaving tasklet */
382 enable_irq(pc->irq);
383 }
384
mtk_cqdma_irq(int irq,void * devid)385 static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
386 {
387 struct mtk_cqdma_device *cqdma = devid;
388 irqreturn_t ret = IRQ_NONE;
389 bool schedule_tasklet = false;
390 u32 i;
391
392 /* clear interrupt flags for each PC */
393 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
394 spin_lock(&cqdma->pc[i]->lock);
395 if (mtk_dma_read(cqdma->pc[i],
396 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
397 /* clear interrupt */
398 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
399 MTK_CQDMA_INT_FLAG_BIT);
400
401 schedule_tasklet = true;
402 ret = IRQ_HANDLED;
403 }
404 spin_unlock(&cqdma->pc[i]->lock);
405
406 if (schedule_tasklet) {
407 /* disable interrupt */
408 disable_irq_nosync(cqdma->pc[i]->irq);
409
410 /* schedule the tasklet to handle the transactions */
411 tasklet_schedule(&cqdma->pc[i]->tasklet);
412 }
413 }
414
415 return ret;
416 }
417
mtk_cqdma_find_active_desc(struct dma_chan * c,dma_cookie_t cookie)418 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
419 dma_cookie_t cookie)
420 {
421 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
422 struct virt_dma_desc *vd;
423
424 list_for_each_entry(vd, &cvc->pc->queue, node)
425 if (vd->tx.cookie == cookie) {
426 return vd;
427 }
428
429 list_for_each_entry(vd, &cvc->vc.desc_issued, node)
430 if (vd->tx.cookie == cookie)
431 return vd;
432
433 return NULL;
434 }
435
mtk_cqdma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)436 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
437 dma_cookie_t cookie,
438 struct dma_tx_state *txstate)
439 {
440 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
441 struct mtk_cqdma_vdesc *cvd;
442 struct virt_dma_desc *vd;
443 enum dma_status ret;
444 unsigned long flags;
445 size_t bytes = 0;
446
447 ret = dma_cookie_status(c, cookie, txstate);
448 if (ret == DMA_COMPLETE || !txstate)
449 return ret;
450
451 spin_lock_irqsave(&cvc->pc->lock, flags);
452 spin_lock_irqsave(&cvc->vc.lock, flags);
453 vd = mtk_cqdma_find_active_desc(c, cookie);
454 spin_unlock_irqrestore(&cvc->vc.lock, flags);
455 spin_unlock_irqrestore(&cvc->pc->lock, flags);
456
457 if (vd) {
458 cvd = to_cqdma_vdesc(vd);
459 bytes = cvd->residue;
460 }
461
462 dma_set_residue(txstate, bytes);
463
464 return ret;
465 }
466
mtk_cqdma_issue_pending(struct dma_chan * c)467 static void mtk_cqdma_issue_pending(struct dma_chan *c)
468 {
469 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
470 unsigned long pc_flags;
471 unsigned long vc_flags;
472
473 /* acquire PC's lock before VS's lock for lock dependency in tasklet */
474 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
475 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
476
477 if (vchan_issue_pending(&cvc->vc))
478 mtk_cqdma_issue_vchan_pending(cvc);
479
480 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
481 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
482 }
483
484 static struct dma_async_tx_descriptor *
mtk_cqdma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)485 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
486 dma_addr_t src, size_t len, unsigned long flags)
487 {
488 struct mtk_cqdma_vdesc **cvd;
489 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
490 size_t i, tlen, nr_vd;
491
492 /*
493 * In the case that trsanction length is larger than the
494 * DMA engine supports, a single memcpy transaction needs
495 * to be separated into several DMA transactions.
496 * Each DMA transaction would be described by a CVD,
497 * and the first one is referred as the parent CVD,
498 * while the others are child CVDs.
499 * The parent CVD's tx descriptor is the only tx descriptor
500 * returned to the DMA user, and it should not be completed
501 * until all the child CVDs completed.
502 */
503 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
504 cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
505 if (!cvd)
506 return NULL;
507
508 for (i = 0; i < nr_vd; ++i) {
509 cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
510 if (!cvd[i]) {
511 for (; i > 0; --i)
512 kfree(cvd[i - 1]);
513 return NULL;
514 }
515
516 /* setup dma channel */
517 cvd[i]->ch = c;
518
519 /* setup source, destination, and length */
520 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
521 cvd[i]->len = tlen;
522 cvd[i]->src = src;
523 cvd[i]->dest = dest;
524
525 /* setup tx descriptor */
526 tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
527 tx->next = NULL;
528
529 if (!i) {
530 cvd[0]->residue = len;
531 } else {
532 prev_tx->next = tx;
533 cvd[i]->residue = tlen;
534 }
535
536 cvd[i]->parent = cvd[0];
537
538 /* update the src, dest, len, prev_tx for the next CVD */
539 src += tlen;
540 dest += tlen;
541 len -= tlen;
542 prev_tx = tx;
543 }
544
545 return &cvd[0]->vd.tx;
546 }
547
mtk_cqdma_free_inactive_desc(struct dma_chan * c)548 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
549 {
550 struct virt_dma_chan *vc = to_virt_chan(c);
551 unsigned long flags;
552 LIST_HEAD(head);
553
554 /*
555 * set desc_allocated, desc_submitted,
556 * and desc_issued as the candicates to be freed
557 */
558 spin_lock_irqsave(&vc->lock, flags);
559 list_splice_tail_init(&vc->desc_allocated, &head);
560 list_splice_tail_init(&vc->desc_submitted, &head);
561 list_splice_tail_init(&vc->desc_issued, &head);
562 spin_unlock_irqrestore(&vc->lock, flags);
563
564 /* free descriptor lists */
565 vchan_dma_desc_free_list(vc, &head);
566 }
567
mtk_cqdma_free_active_desc(struct dma_chan * c)568 static void mtk_cqdma_free_active_desc(struct dma_chan *c)
569 {
570 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
571 bool sync_needed = false;
572 unsigned long pc_flags;
573 unsigned long vc_flags;
574
575 /* acquire PC's lock first due to lock dependency in dma ISR */
576 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
577 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
578
579 /* synchronization is required if this VC is active */
580 if (mtk_cqdma_is_vchan_active(cvc)) {
581 cvc->issue_synchronize = true;
582 sync_needed = true;
583 }
584
585 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
586 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
587
588 /* waiting for the completion of this VC */
589 if (sync_needed)
590 wait_for_completion(&cvc->issue_completion);
591
592 /* free all descriptors in list desc_completed */
593 vchan_synchronize(&cvc->vc);
594
595 WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
596 "Desc pending still in list desc_completed\n");
597 }
598
mtk_cqdma_terminate_all(struct dma_chan * c)599 static int mtk_cqdma_terminate_all(struct dma_chan *c)
600 {
601 /* free descriptors not processed yet by hardware */
602 mtk_cqdma_free_inactive_desc(c);
603
604 /* free descriptors being processed by hardware */
605 mtk_cqdma_free_active_desc(c);
606
607 return 0;
608 }
609
mtk_cqdma_alloc_chan_resources(struct dma_chan * c)610 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
611 {
612 struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
613 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
614 struct mtk_cqdma_pchan *pc = NULL;
615 u32 i, min_refcnt = U32_MAX, refcnt;
616 unsigned long flags;
617
618 /* allocate PC with the minimum refcount */
619 for (i = 0; i < cqdma->dma_channels; ++i) {
620 refcnt = refcount_read(&cqdma->pc[i]->refcnt);
621 if (refcnt < min_refcnt) {
622 pc = cqdma->pc[i];
623 min_refcnt = refcnt;
624 }
625 }
626
627 if (!pc)
628 return -ENOSPC;
629
630 spin_lock_irqsave(&pc->lock, flags);
631
632 if (!refcount_read(&pc->refcnt)) {
633 /* allocate PC when the refcount is zero */
634 mtk_cqdma_hard_reset(pc);
635
636 /* enable interrupt for this PC */
637 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
638
639 /*
640 * refcount_inc would complain increment on 0; use-after-free.
641 * Thus, we need to explicitly set it as 1 initially.
642 */
643 refcount_set(&pc->refcnt, 1);
644 } else {
645 refcount_inc(&pc->refcnt);
646 }
647
648 spin_unlock_irqrestore(&pc->lock, flags);
649
650 vc->pc = pc;
651
652 return 0;
653 }
654
mtk_cqdma_free_chan_resources(struct dma_chan * c)655 static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
656 {
657 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
658 unsigned long flags;
659
660 /* free all descriptors in all lists on the VC */
661 mtk_cqdma_terminate_all(c);
662
663 spin_lock_irqsave(&cvc->pc->lock, flags);
664
665 /* PC is not freed until there is no VC mapped to it */
666 if (refcount_dec_and_test(&cvc->pc->refcnt)) {
667 /* start the flush operation and stop the engine */
668 mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
669
670 /* wait for the completion of flush operation */
671 if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
672 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
673
674 /* clear the flush bit and interrupt flag */
675 mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
676 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
677 MTK_CQDMA_INT_FLAG_BIT);
678
679 /* disable interrupt for this PC */
680 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
681 }
682
683 spin_unlock_irqrestore(&cvc->pc->lock, flags);
684 }
685
mtk_cqdma_hw_init(struct mtk_cqdma_device * cqdma)686 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
687 {
688 unsigned long flags;
689 int err;
690 u32 i;
691
692 pm_runtime_enable(cqdma2dev(cqdma));
693 pm_runtime_get_sync(cqdma2dev(cqdma));
694
695 err = clk_prepare_enable(cqdma->clk);
696
697 if (err) {
698 pm_runtime_put_sync(cqdma2dev(cqdma));
699 pm_runtime_disable(cqdma2dev(cqdma));
700 return err;
701 }
702
703 /* reset all PCs */
704 for (i = 0; i < cqdma->dma_channels; ++i) {
705 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
706 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
707 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
708 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
709
710 clk_disable_unprepare(cqdma->clk);
711 pm_runtime_put_sync(cqdma2dev(cqdma));
712 pm_runtime_disable(cqdma2dev(cqdma));
713 return -EINVAL;
714 }
715 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
716 }
717
718 return 0;
719 }
720
mtk_cqdma_hw_deinit(struct mtk_cqdma_device * cqdma)721 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
722 {
723 unsigned long flags;
724 u32 i;
725
726 /* reset all PCs */
727 for (i = 0; i < cqdma->dma_channels; ++i) {
728 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
729 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
730 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
731 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
732 }
733
734 clk_disable_unprepare(cqdma->clk);
735
736 pm_runtime_put_sync(cqdma2dev(cqdma));
737 pm_runtime_disable(cqdma2dev(cqdma));
738 }
739
740 static const struct of_device_id mtk_cqdma_match[] = {
741 { .compatible = "mediatek,mt6765-cqdma" },
742 { /* sentinel */ }
743 };
744 MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
745
mtk_cqdma_probe(struct platform_device * pdev)746 static int mtk_cqdma_probe(struct platform_device *pdev)
747 {
748 struct mtk_cqdma_device *cqdma;
749 struct mtk_cqdma_vchan *vc;
750 struct dma_device *dd;
751 int err;
752 u32 i;
753
754 cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
755 if (!cqdma)
756 return -ENOMEM;
757
758 dd = &cqdma->ddev;
759
760 cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
761 if (IS_ERR(cqdma->clk)) {
762 dev_err(&pdev->dev, "No clock for %s\n",
763 dev_name(&pdev->dev));
764 return PTR_ERR(cqdma->clk);
765 }
766
767 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
768
769 dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
770 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
771 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
772 dd->device_tx_status = mtk_cqdma_tx_status;
773 dd->device_issue_pending = mtk_cqdma_issue_pending;
774 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
775 dd->device_terminate_all = mtk_cqdma_terminate_all;
776 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
777 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
778 dd->directions = BIT(DMA_MEM_TO_MEM);
779 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
780 dd->dev = &pdev->dev;
781 INIT_LIST_HEAD(&dd->channels);
782
783 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
784 "dma-requests",
785 &cqdma->dma_requests)) {
786 dev_info(&pdev->dev,
787 "Using %u as missing dma-requests property\n",
788 MTK_CQDMA_NR_VCHANS);
789
790 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
791 }
792
793 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
794 "dma-channels",
795 &cqdma->dma_channels)) {
796 dev_info(&pdev->dev,
797 "Using %u as missing dma-channels property\n",
798 MTK_CQDMA_NR_PCHANS);
799
800 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
801 }
802
803 cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
804 sizeof(*cqdma->pc), GFP_KERNEL);
805 if (!cqdma->pc)
806 return -ENOMEM;
807
808 /* initialization for PCs */
809 for (i = 0; i < cqdma->dma_channels; ++i) {
810 cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
811 sizeof(**cqdma->pc), GFP_KERNEL);
812 if (!cqdma->pc[i])
813 return -ENOMEM;
814
815 INIT_LIST_HEAD(&cqdma->pc[i]->queue);
816 spin_lock_init(&cqdma->pc[i]->lock);
817 refcount_set(&cqdma->pc[i]->refcnt, 0);
818 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
819 if (IS_ERR(cqdma->pc[i]->base))
820 return PTR_ERR(cqdma->pc[i]->base);
821
822 /* allocate IRQ resource */
823 err = platform_get_irq(pdev, i);
824 if (err < 0)
825 return err;
826 cqdma->pc[i]->irq = err;
827
828 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
829 mtk_cqdma_irq, 0, dev_name(&pdev->dev),
830 cqdma);
831 if (err) {
832 dev_err(&pdev->dev,
833 "request_irq failed with err %d\n", err);
834 return -EINVAL;
835 }
836 }
837
838 /* allocate resource for VCs */
839 cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
840 sizeof(*cqdma->vc), GFP_KERNEL);
841 if (!cqdma->vc)
842 return -ENOMEM;
843
844 for (i = 0; i < cqdma->dma_requests; i++) {
845 vc = &cqdma->vc[i];
846 vc->vc.desc_free = mtk_cqdma_vdesc_free;
847 vchan_init(&vc->vc, dd);
848 init_completion(&vc->issue_completion);
849 }
850
851 err = dma_async_device_register(dd);
852 if (err)
853 return err;
854
855 err = of_dma_controller_register(pdev->dev.of_node,
856 of_dma_xlate_by_chan_id, cqdma);
857 if (err) {
858 dev_err(&pdev->dev,
859 "MediaTek CQDMA OF registration failed %d\n", err);
860 goto err_unregister;
861 }
862
863 err = mtk_cqdma_hw_init(cqdma);
864 if (err) {
865 dev_err(&pdev->dev,
866 "MediaTek CQDMA HW initialization failed %d\n", err);
867 goto err_unregister;
868 }
869
870 platform_set_drvdata(pdev, cqdma);
871
872 /* initialize tasklet for each PC */
873 for (i = 0; i < cqdma->dma_channels; ++i)
874 tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
875
876 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
877
878 return 0;
879
880 err_unregister:
881 dma_async_device_unregister(dd);
882
883 return err;
884 }
885
mtk_cqdma_remove(struct platform_device * pdev)886 static void mtk_cqdma_remove(struct platform_device *pdev)
887 {
888 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
889 struct mtk_cqdma_vchan *vc;
890 unsigned long flags;
891 int i;
892
893 /* kill VC task */
894 for (i = 0; i < cqdma->dma_requests; i++) {
895 vc = &cqdma->vc[i];
896
897 list_del(&vc->vc.chan.device_node);
898 tasklet_kill(&vc->vc.task);
899 }
900
901 /* disable interrupt */
902 for (i = 0; i < cqdma->dma_channels; i++) {
903 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
904 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
905 MTK_CQDMA_INT_EN_BIT);
906 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
907
908 /* Waits for any pending IRQ handlers to complete */
909 synchronize_irq(cqdma->pc[i]->irq);
910
911 tasklet_kill(&cqdma->pc[i]->tasklet);
912 }
913
914 /* disable hardware */
915 mtk_cqdma_hw_deinit(cqdma);
916
917 dma_async_device_unregister(&cqdma->ddev);
918 of_dma_controller_free(pdev->dev.of_node);
919 }
920
921 static struct platform_driver mtk_cqdma_driver = {
922 .probe = mtk_cqdma_probe,
923 .remove = mtk_cqdma_remove,
924 .driver = {
925 .name = KBUILD_MODNAME,
926 .of_match_table = mtk_cqdma_match,
927 },
928 };
929 module_platform_driver(mtk_cqdma_driver);
930
931 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
932 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
933 MODULE_LICENSE("GPL v2");
934