1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2018-2019 MediaTek Inc.
3
4 /*
5 * Driver for MediaTek Command-Queue DMA Controller
6 *
7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
8 *
9 */
10
11 #include <linux/bitops.h>
12 #include <linux/clk.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/iopoll.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_dma.h>
22 #include <linux/platform_device.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/refcount.h>
25 #include <linux/slab.h>
26
27 #include "../virt-dma.h"
28
29 #define MTK_CQDMA_USEC_POLL 10
30 #define MTK_CQDMA_TIMEOUT_POLL 1000
31 #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
32 #define MTK_CQDMA_ALIGN_SIZE 1
33
34 /* The default number of virtual channel */
35 #define MTK_CQDMA_NR_VCHANS 32
36
37 /* The default number of physical channel */
38 #define MTK_CQDMA_NR_PCHANS 3
39
40 /* Registers for underlying dma manipulation */
41 #define MTK_CQDMA_INT_FLAG 0x0
42 #define MTK_CQDMA_INT_EN 0x4
43 #define MTK_CQDMA_EN 0x8
44 #define MTK_CQDMA_RESET 0xc
45 #define MTK_CQDMA_FLUSH 0x14
46 #define MTK_CQDMA_SRC 0x1c
47 #define MTK_CQDMA_DST 0x20
48 #define MTK_CQDMA_LEN1 0x24
49 #define MTK_CQDMA_LEN2 0x28
50 #define MTK_CQDMA_SRC2 0x60
51 #define MTK_CQDMA_DST2 0x64
52
53 /* Registers setting */
54 #define MTK_CQDMA_EN_BIT BIT(0)
55 #define MTK_CQDMA_INT_FLAG_BIT BIT(0)
56 #define MTK_CQDMA_INT_EN_BIT BIT(0)
57 #define MTK_CQDMA_FLUSH_BIT BIT(0)
58
59 #define MTK_CQDMA_WARM_RST_BIT BIT(0)
60 #define MTK_CQDMA_HARD_RST_BIT BIT(1)
61
62 #define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
63 #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
64 #define MTK_CQDMA_ADDR2_SHFIT (32)
65
66 /**
67 * struct mtk_cqdma_vdesc - The struct holding info describing virtual
68 * descriptor (CVD)
69 * @vd: An instance for struct virt_dma_desc
70 * @len: The total data size device wants to move
71 * @residue: The remaining data size device will move
72 * @dest: The destination address device wants to move to
73 * @src: The source address device wants to move from
74 * @ch: The pointer to the corresponding dma channel
75 * @node: The lise_head struct to build link-list for VDs
76 * @parent: The pointer to the parent CVD
77 */
78 struct mtk_cqdma_vdesc {
79 struct virt_dma_desc vd;
80 size_t len;
81 size_t residue;
82 dma_addr_t dest;
83 dma_addr_t src;
84 struct dma_chan *ch;
85
86 struct list_head node;
87 struct mtk_cqdma_vdesc *parent;
88 };
89
90 /**
91 * struct mtk_cqdma_pchan - The struct holding info describing physical
92 * channel (PC)
93 * @queue: Queue for the PDs issued to this PC
94 * @base: The mapped register I/O base of this PC
95 * @irq: The IRQ that this PC are using
96 * @refcnt: Track how many VCs are using this PC
97 * @tasklet: Tasklet for this PC
98 * @lock: Lock protect agaisting multiple VCs access PC
99 */
100 struct mtk_cqdma_pchan {
101 struct list_head queue;
102 void __iomem *base;
103 u32 irq;
104
105 refcount_t refcnt;
106
107 struct tasklet_struct tasklet;
108
109 /* lock to protect PC */
110 spinlock_t lock;
111 };
112
113 /**
114 * struct mtk_cqdma_vchan - The struct holding info describing virtual
115 * channel (VC)
116 * @vc: An instance for struct virt_dma_chan
117 * @pc: The pointer to the underlying PC
118 * @issue_completion: The wait for all issued descriptors completited
119 * @issue_synchronize: Bool indicating channel synchronization starts
120 */
121 struct mtk_cqdma_vchan {
122 struct virt_dma_chan vc;
123 struct mtk_cqdma_pchan *pc;
124 struct completion issue_completion;
125 bool issue_synchronize;
126 };
127
128 /**
129 * struct mtk_cqdma_device - The struct holding info describing CQDMA
130 * device
131 * @ddev: An instance for struct dma_device
132 * @clk: The clock that device internal is using
133 * @dma_requests: The number of VCs the device supports to
134 * @dma_channels: The number of PCs the device supports to
135 * @vc: The pointer to all available VCs
136 * @pc: The pointer to all the underlying PCs
137 */
138 struct mtk_cqdma_device {
139 struct dma_device ddev;
140 struct clk *clk;
141
142 u32 dma_requests;
143 u32 dma_channels;
144 struct mtk_cqdma_vchan *vc;
145 struct mtk_cqdma_pchan **pc;
146 };
147
to_cqdma_dev(struct dma_chan * chan)148 static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
149 {
150 return container_of(chan->device, struct mtk_cqdma_device, ddev);
151 }
152
to_cqdma_vchan(struct dma_chan * chan)153 static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
154 {
155 return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
156 }
157
to_cqdma_vdesc(struct virt_dma_desc * vd)158 static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
159 {
160 return container_of(vd, struct mtk_cqdma_vdesc, vd);
161 }
162
cqdma2dev(struct mtk_cqdma_device * cqdma)163 static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
164 {
165 return cqdma->ddev.dev;
166 }
167
mtk_dma_read(struct mtk_cqdma_pchan * pc,u32 reg)168 static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
169 {
170 return readl(pc->base + reg);
171 }
172
mtk_dma_write(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)173 static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
174 {
175 writel_relaxed(val, pc->base + reg);
176 }
177
mtk_dma_rmw(struct mtk_cqdma_pchan * pc,u32 reg,u32 mask,u32 set)178 static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
179 u32 mask, u32 set)
180 {
181 u32 val;
182
183 val = mtk_dma_read(pc, reg);
184 val &= ~mask;
185 val |= set;
186 mtk_dma_write(pc, reg, val);
187 }
188
mtk_dma_set(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)189 static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
190 {
191 mtk_dma_rmw(pc, reg, 0, val);
192 }
193
mtk_dma_clr(struct mtk_cqdma_pchan * pc,u32 reg,u32 val)194 static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
195 {
196 mtk_dma_rmw(pc, reg, val, 0);
197 }
198
mtk_cqdma_vdesc_free(struct virt_dma_desc * vd)199 static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
200 {
201 kfree(to_cqdma_vdesc(vd));
202 }
203
mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan * pc,bool atomic)204 static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
205 {
206 u32 status = 0;
207
208 if (!atomic)
209 return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
210 status,
211 !(status & MTK_CQDMA_EN_BIT),
212 MTK_CQDMA_USEC_POLL,
213 MTK_CQDMA_TIMEOUT_POLL);
214
215 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
216 status,
217 !(status & MTK_CQDMA_EN_BIT),
218 MTK_CQDMA_USEC_POLL,
219 MTK_CQDMA_TIMEOUT_POLL);
220 }
221
mtk_cqdma_hard_reset(struct mtk_cqdma_pchan * pc)222 static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
223 {
224 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
225 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226
227 return mtk_cqdma_poll_engine_done(pc, true);
228 }
229
mtk_cqdma_start(struct mtk_cqdma_pchan * pc,struct mtk_cqdma_vdesc * cvd)230 static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
231 struct mtk_cqdma_vdesc *cvd)
232 {
233 /* wait for the previous transaction done */
234 if (mtk_cqdma_poll_engine_done(pc, true) < 0)
235 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
236
237 /* warm reset the dma engine for the new transaction */
238 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
239 if (mtk_cqdma_poll_engine_done(pc, true) < 0)
240 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
241
242 /* setup the source */
243 mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
244 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
245 mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
246 #else
247 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
248 #endif
249
250 /* setup the destination */
251 mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
252 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
253 mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
254 #else
255 mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
256 #endif
257
258 /* setup the length */
259 mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
260
261 /* start dma engine */
262 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
263 }
264
mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan * cvc)265 static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
266 {
267 struct virt_dma_desc *vd, *vd2;
268 struct mtk_cqdma_pchan *pc = cvc->pc;
269 struct mtk_cqdma_vdesc *cvd;
270 bool trigger_engine = false;
271
272 lockdep_assert_held(&cvc->vc.lock);
273 lockdep_assert_held(&pc->lock);
274
275 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
276 /* need to trigger dma engine if PC's queue is empty */
277 if (list_empty(&pc->queue))
278 trigger_engine = true;
279
280 cvd = to_cqdma_vdesc(vd);
281
282 /* add VD into PC's queue */
283 list_add_tail(&cvd->node, &pc->queue);
284
285 /* start the dma engine */
286 if (trigger_engine)
287 mtk_cqdma_start(pc, cvd);
288
289 /* remove VD from list desc_issued */
290 list_del(&vd->node);
291 }
292 }
293
294 /*
295 * return true if this VC is active,
296 * meaning that there are VDs under processing by the PC
297 */
mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan * cvc)298 static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
299 {
300 struct mtk_cqdma_vdesc *cvd;
301
302 list_for_each_entry(cvd, &cvc->pc->queue, node)
303 if (cvc == to_cqdma_vchan(cvd->ch))
304 return true;
305
306 return false;
307 }
308
309 /*
310 * return the pointer of the CVD that is just consumed by the PC
311 */
312 static struct mtk_cqdma_vdesc
mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan * pc)313 *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
314 {
315 struct mtk_cqdma_vchan *cvc;
316 struct mtk_cqdma_vdesc *cvd, *ret = NULL;
317
318 /* consume a CVD from PC's queue */
319 cvd = list_first_entry_or_null(&pc->queue,
320 struct mtk_cqdma_vdesc, node);
321 if (unlikely(!cvd || !cvd->parent))
322 return NULL;
323
324 cvc = to_cqdma_vchan(cvd->ch);
325 ret = cvd;
326
327 /* update residue of the parent CVD */
328 cvd->parent->residue -= cvd->len;
329
330 /* delete CVD from PC's queue */
331 list_del(&cvd->node);
332
333 spin_lock(&cvc->vc.lock);
334
335 /* check whether all the child CVDs completed */
336 if (!cvd->parent->residue) {
337 /* add the parent VD into list desc_completed */
338 vchan_cookie_complete(&cvd->parent->vd);
339
340 /* setup completion if this VC is under synchronization */
341 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
342 complete(&cvc->issue_completion);
343 cvc->issue_synchronize = false;
344 }
345 }
346
347 spin_unlock(&cvc->vc.lock);
348
349 /* start transaction for next CVD in the queue */
350 cvd = list_first_entry_or_null(&pc->queue,
351 struct mtk_cqdma_vdesc, node);
352 if (cvd)
353 mtk_cqdma_start(pc, cvd);
354
355 return ret;
356 }
357
mtk_cqdma_tasklet_cb(struct tasklet_struct * t)358 static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
359 {
360 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
361 struct mtk_cqdma_vdesc *cvd = NULL;
362 unsigned long flags;
363
364 spin_lock_irqsave(&pc->lock, flags);
365 /* consume the queue */
366 cvd = mtk_cqdma_consume_work_queue(pc);
367 spin_unlock_irqrestore(&pc->lock, flags);
368
369 /* submit the next CVD */
370 if (cvd) {
371 dma_run_dependencies(&cvd->vd.tx);
372
373 /*
374 * free child CVD after completion.
375 * the parent CVD would be freed with desc_free by user.
376 */
377 if (cvd->parent != cvd)
378 kfree(cvd);
379 }
380
381 /* re-enable interrupt before leaving tasklet */
382 enable_irq(pc->irq);
383 }
384
mtk_cqdma_irq(int irq,void * devid)385 static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
386 {
387 struct mtk_cqdma_device *cqdma = devid;
388 irqreturn_t ret = IRQ_NONE;
389 bool schedule_tasklet = false;
390 u32 i;
391
392 /* clear interrupt flags for each PC */
393 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
394 spin_lock(&cqdma->pc[i]->lock);
395 if (mtk_dma_read(cqdma->pc[i],
396 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
397 /* clear interrupt */
398 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
399 MTK_CQDMA_INT_FLAG_BIT);
400
401 schedule_tasklet = true;
402 ret = IRQ_HANDLED;
403 }
404 spin_unlock(&cqdma->pc[i]->lock);
405
406 if (schedule_tasklet) {
407 /* disable interrupt */
408 disable_irq_nosync(cqdma->pc[i]->irq);
409
410 /* schedule the tasklet to handle the transactions */
411 tasklet_schedule(&cqdma->pc[i]->tasklet);
412 }
413 }
414
415 return ret;
416 }
417
mtk_cqdma_find_active_desc(struct dma_chan * c,dma_cookie_t cookie)418 static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
419 dma_cookie_t cookie)
420 {
421 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
422 struct virt_dma_desc *vd;
423 unsigned long flags;
424
425 spin_lock_irqsave(&cvc->pc->lock, flags);
426 list_for_each_entry(vd, &cvc->pc->queue, node)
427 if (vd->tx.cookie == cookie) {
428 spin_unlock_irqrestore(&cvc->pc->lock, flags);
429 return vd;
430 }
431 spin_unlock_irqrestore(&cvc->pc->lock, flags);
432
433 list_for_each_entry(vd, &cvc->vc.desc_issued, node)
434 if (vd->tx.cookie == cookie)
435 return vd;
436
437 return NULL;
438 }
439
mtk_cqdma_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)440 static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
441 dma_cookie_t cookie,
442 struct dma_tx_state *txstate)
443 {
444 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
445 struct mtk_cqdma_vdesc *cvd;
446 struct virt_dma_desc *vd;
447 enum dma_status ret;
448 unsigned long flags;
449 size_t bytes = 0;
450
451 ret = dma_cookie_status(c, cookie, txstate);
452 if (ret == DMA_COMPLETE || !txstate)
453 return ret;
454
455 spin_lock_irqsave(&cvc->vc.lock, flags);
456 vd = mtk_cqdma_find_active_desc(c, cookie);
457 spin_unlock_irqrestore(&cvc->vc.lock, flags);
458
459 if (vd) {
460 cvd = to_cqdma_vdesc(vd);
461 bytes = cvd->residue;
462 }
463
464 dma_set_residue(txstate, bytes);
465
466 return ret;
467 }
468
mtk_cqdma_issue_pending(struct dma_chan * c)469 static void mtk_cqdma_issue_pending(struct dma_chan *c)
470 {
471 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
472 unsigned long pc_flags;
473 unsigned long vc_flags;
474
475 /* acquire PC's lock before VS's lock for lock dependency in tasklet */
476 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
477 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
478
479 if (vchan_issue_pending(&cvc->vc))
480 mtk_cqdma_issue_vchan_pending(cvc);
481
482 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
483 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
484 }
485
486 static struct dma_async_tx_descriptor *
mtk_cqdma_prep_dma_memcpy(struct dma_chan * c,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)487 mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
488 dma_addr_t src, size_t len, unsigned long flags)
489 {
490 struct mtk_cqdma_vdesc **cvd;
491 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
492 size_t i, tlen, nr_vd;
493
494 /*
495 * In the case that trsanction length is larger than the
496 * DMA engine supports, a single memcpy transaction needs
497 * to be separated into several DMA transactions.
498 * Each DMA transaction would be described by a CVD,
499 * and the first one is referred as the parent CVD,
500 * while the others are child CVDs.
501 * The parent CVD's tx descriptor is the only tx descriptor
502 * returned to the DMA user, and it should not be completed
503 * until all the child CVDs completed.
504 */
505 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
506 cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
507 if (!cvd)
508 return NULL;
509
510 for (i = 0; i < nr_vd; ++i) {
511 cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
512 if (!cvd[i]) {
513 for (; i > 0; --i)
514 kfree(cvd[i - 1]);
515 return NULL;
516 }
517
518 /* setup dma channel */
519 cvd[i]->ch = c;
520
521 /* setup source, destination, and length */
522 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
523 cvd[i]->len = tlen;
524 cvd[i]->src = src;
525 cvd[i]->dest = dest;
526
527 /* setup tx descriptor */
528 tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
529 tx->next = NULL;
530
531 if (!i) {
532 cvd[0]->residue = len;
533 } else {
534 prev_tx->next = tx;
535 cvd[i]->residue = tlen;
536 }
537
538 cvd[i]->parent = cvd[0];
539
540 /* update the src, dest, len, prev_tx for the next CVD */
541 src += tlen;
542 dest += tlen;
543 len -= tlen;
544 prev_tx = tx;
545 }
546
547 return &cvd[0]->vd.tx;
548 }
549
mtk_cqdma_free_inactive_desc(struct dma_chan * c)550 static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
551 {
552 struct virt_dma_chan *vc = to_virt_chan(c);
553 unsigned long flags;
554 LIST_HEAD(head);
555
556 /*
557 * set desc_allocated, desc_submitted,
558 * and desc_issued as the candicates to be freed
559 */
560 spin_lock_irqsave(&vc->lock, flags);
561 list_splice_tail_init(&vc->desc_allocated, &head);
562 list_splice_tail_init(&vc->desc_submitted, &head);
563 list_splice_tail_init(&vc->desc_issued, &head);
564 spin_unlock_irqrestore(&vc->lock, flags);
565
566 /* free descriptor lists */
567 vchan_dma_desc_free_list(vc, &head);
568 }
569
mtk_cqdma_free_active_desc(struct dma_chan * c)570 static void mtk_cqdma_free_active_desc(struct dma_chan *c)
571 {
572 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
573 bool sync_needed = false;
574 unsigned long pc_flags;
575 unsigned long vc_flags;
576
577 /* acquire PC's lock first due to lock dependency in dma ISR */
578 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
579 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
580
581 /* synchronization is required if this VC is active */
582 if (mtk_cqdma_is_vchan_active(cvc)) {
583 cvc->issue_synchronize = true;
584 sync_needed = true;
585 }
586
587 spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
588 spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
589
590 /* waiting for the completion of this VC */
591 if (sync_needed)
592 wait_for_completion(&cvc->issue_completion);
593
594 /* free all descriptors in list desc_completed */
595 vchan_synchronize(&cvc->vc);
596
597 WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
598 "Desc pending still in list desc_completed\n");
599 }
600
mtk_cqdma_terminate_all(struct dma_chan * c)601 static int mtk_cqdma_terminate_all(struct dma_chan *c)
602 {
603 /* free descriptors not processed yet by hardware */
604 mtk_cqdma_free_inactive_desc(c);
605
606 /* free descriptors being processed by hardware */
607 mtk_cqdma_free_active_desc(c);
608
609 return 0;
610 }
611
mtk_cqdma_alloc_chan_resources(struct dma_chan * c)612 static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
613 {
614 struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
615 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
616 struct mtk_cqdma_pchan *pc = NULL;
617 u32 i, min_refcnt = U32_MAX, refcnt;
618 unsigned long flags;
619
620 /* allocate PC with the minimum refcount */
621 for (i = 0; i < cqdma->dma_channels; ++i) {
622 refcnt = refcount_read(&cqdma->pc[i]->refcnt);
623 if (refcnt < min_refcnt) {
624 pc = cqdma->pc[i];
625 min_refcnt = refcnt;
626 }
627 }
628
629 if (!pc)
630 return -ENOSPC;
631
632 spin_lock_irqsave(&pc->lock, flags);
633
634 if (!refcount_read(&pc->refcnt)) {
635 /* allocate PC when the refcount is zero */
636 mtk_cqdma_hard_reset(pc);
637
638 /* enable interrupt for this PC */
639 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
640
641 /*
642 * refcount_inc would complain increment on 0; use-after-free.
643 * Thus, we need to explicitly set it as 1 initially.
644 */
645 refcount_set(&pc->refcnt, 1);
646 } else {
647 refcount_inc(&pc->refcnt);
648 }
649
650 spin_unlock_irqrestore(&pc->lock, flags);
651
652 vc->pc = pc;
653
654 return 0;
655 }
656
mtk_cqdma_free_chan_resources(struct dma_chan * c)657 static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
658 {
659 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
660 unsigned long flags;
661
662 /* free all descriptors in all lists on the VC */
663 mtk_cqdma_terminate_all(c);
664
665 spin_lock_irqsave(&cvc->pc->lock, flags);
666
667 /* PC is not freed until there is no VC mapped to it */
668 if (refcount_dec_and_test(&cvc->pc->refcnt)) {
669 /* start the flush operation and stop the engine */
670 mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
671
672 /* wait for the completion of flush operation */
673 if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
674 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
675
676 /* clear the flush bit and interrupt flag */
677 mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
678 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
679 MTK_CQDMA_INT_FLAG_BIT);
680
681 /* disable interrupt for this PC */
682 mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
683 }
684
685 spin_unlock_irqrestore(&cvc->pc->lock, flags);
686 }
687
mtk_cqdma_hw_init(struct mtk_cqdma_device * cqdma)688 static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
689 {
690 unsigned long flags;
691 int err;
692 u32 i;
693
694 pm_runtime_enable(cqdma2dev(cqdma));
695 pm_runtime_get_sync(cqdma2dev(cqdma));
696
697 err = clk_prepare_enable(cqdma->clk);
698
699 if (err) {
700 pm_runtime_put_sync(cqdma2dev(cqdma));
701 pm_runtime_disable(cqdma2dev(cqdma));
702 return err;
703 }
704
705 /* reset all PCs */
706 for (i = 0; i < cqdma->dma_channels; ++i) {
707 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
708 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
709 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
710 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
711
712 clk_disable_unprepare(cqdma->clk);
713 pm_runtime_put_sync(cqdma2dev(cqdma));
714 pm_runtime_disable(cqdma2dev(cqdma));
715 return -EINVAL;
716 }
717 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
718 }
719
720 return 0;
721 }
722
mtk_cqdma_hw_deinit(struct mtk_cqdma_device * cqdma)723 static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
724 {
725 unsigned long flags;
726 u32 i;
727
728 /* reset all PCs */
729 for (i = 0; i < cqdma->dma_channels; ++i) {
730 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
731 if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
732 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
733 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
734 }
735
736 clk_disable_unprepare(cqdma->clk);
737
738 pm_runtime_put_sync(cqdma2dev(cqdma));
739 pm_runtime_disable(cqdma2dev(cqdma));
740 }
741
742 static const struct of_device_id mtk_cqdma_match[] = {
743 { .compatible = "mediatek,mt6765-cqdma" },
744 { /* sentinel */ }
745 };
746 MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
747
mtk_cqdma_probe(struct platform_device * pdev)748 static int mtk_cqdma_probe(struct platform_device *pdev)
749 {
750 struct mtk_cqdma_device *cqdma;
751 struct mtk_cqdma_vchan *vc;
752 struct dma_device *dd;
753 int err;
754 u32 i;
755
756 cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
757 if (!cqdma)
758 return -ENOMEM;
759
760 dd = &cqdma->ddev;
761
762 cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
763 if (IS_ERR(cqdma->clk)) {
764 dev_err(&pdev->dev, "No clock for %s\n",
765 dev_name(&pdev->dev));
766 return PTR_ERR(cqdma->clk);
767 }
768
769 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
770
771 dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
772 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
773 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
774 dd->device_tx_status = mtk_cqdma_tx_status;
775 dd->device_issue_pending = mtk_cqdma_issue_pending;
776 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
777 dd->device_terminate_all = mtk_cqdma_terminate_all;
778 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
779 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
780 dd->directions = BIT(DMA_MEM_TO_MEM);
781 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
782 dd->dev = &pdev->dev;
783 INIT_LIST_HEAD(&dd->channels);
784
785 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
786 "dma-requests",
787 &cqdma->dma_requests)) {
788 dev_info(&pdev->dev,
789 "Using %u as missing dma-requests property\n",
790 MTK_CQDMA_NR_VCHANS);
791
792 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
793 }
794
795 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
796 "dma-channels",
797 &cqdma->dma_channels)) {
798 dev_info(&pdev->dev,
799 "Using %u as missing dma-channels property\n",
800 MTK_CQDMA_NR_PCHANS);
801
802 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
803 }
804
805 cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
806 sizeof(*cqdma->pc), GFP_KERNEL);
807 if (!cqdma->pc)
808 return -ENOMEM;
809
810 /* initialization for PCs */
811 for (i = 0; i < cqdma->dma_channels; ++i) {
812 cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
813 sizeof(**cqdma->pc), GFP_KERNEL);
814 if (!cqdma->pc[i])
815 return -ENOMEM;
816
817 INIT_LIST_HEAD(&cqdma->pc[i]->queue);
818 spin_lock_init(&cqdma->pc[i]->lock);
819 refcount_set(&cqdma->pc[i]->refcnt, 0);
820 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
821 if (IS_ERR(cqdma->pc[i]->base))
822 return PTR_ERR(cqdma->pc[i]->base);
823
824 /* allocate IRQ resource */
825 err = platform_get_irq(pdev, i);
826 if (err < 0)
827 return err;
828 cqdma->pc[i]->irq = err;
829
830 err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
831 mtk_cqdma_irq, 0, dev_name(&pdev->dev),
832 cqdma);
833 if (err) {
834 dev_err(&pdev->dev,
835 "request_irq failed with err %d\n", err);
836 return -EINVAL;
837 }
838 }
839
840 /* allocate resource for VCs */
841 cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
842 sizeof(*cqdma->vc), GFP_KERNEL);
843 if (!cqdma->vc)
844 return -ENOMEM;
845
846 for (i = 0; i < cqdma->dma_requests; i++) {
847 vc = &cqdma->vc[i];
848 vc->vc.desc_free = mtk_cqdma_vdesc_free;
849 vchan_init(&vc->vc, dd);
850 init_completion(&vc->issue_completion);
851 }
852
853 err = dma_async_device_register(dd);
854 if (err)
855 return err;
856
857 err = of_dma_controller_register(pdev->dev.of_node,
858 of_dma_xlate_by_chan_id, cqdma);
859 if (err) {
860 dev_err(&pdev->dev,
861 "MediaTek CQDMA OF registration failed %d\n", err);
862 goto err_unregister;
863 }
864
865 err = mtk_cqdma_hw_init(cqdma);
866 if (err) {
867 dev_err(&pdev->dev,
868 "MediaTek CQDMA HW initialization failed %d\n", err);
869 goto err_unregister;
870 }
871
872 platform_set_drvdata(pdev, cqdma);
873
874 /* initialize tasklet for each PC */
875 for (i = 0; i < cqdma->dma_channels; ++i)
876 tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
877
878 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
879
880 return 0;
881
882 err_unregister:
883 dma_async_device_unregister(dd);
884
885 return err;
886 }
887
mtk_cqdma_remove(struct platform_device * pdev)888 static void mtk_cqdma_remove(struct platform_device *pdev)
889 {
890 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
891 struct mtk_cqdma_vchan *vc;
892 unsigned long flags;
893 int i;
894
895 /* kill VC task */
896 for (i = 0; i < cqdma->dma_requests; i++) {
897 vc = &cqdma->vc[i];
898
899 list_del(&vc->vc.chan.device_node);
900 tasklet_kill(&vc->vc.task);
901 }
902
903 /* disable interrupt */
904 for (i = 0; i < cqdma->dma_channels; i++) {
905 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
906 mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
907 MTK_CQDMA_INT_EN_BIT);
908 spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
909
910 /* Waits for any pending IRQ handlers to complete */
911 synchronize_irq(cqdma->pc[i]->irq);
912
913 tasklet_kill(&cqdma->pc[i]->tasklet);
914 }
915
916 /* disable hardware */
917 mtk_cqdma_hw_deinit(cqdma);
918
919 dma_async_device_unregister(&cqdma->ddev);
920 of_dma_controller_free(pdev->dev.of_node);
921 }
922
923 static struct platform_driver mtk_cqdma_driver = {
924 .probe = mtk_cqdma_probe,
925 .remove_new = mtk_cqdma_remove,
926 .driver = {
927 .name = KBUILD_MODNAME,
928 .of_match_table = mtk_cqdma_match,
929 },
930 };
931 module_platform_driver(mtk_cqdma_driver);
932
933 MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
934 MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
935 MODULE_LICENSE("GPL v2");
936