1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Renesas RZ/G2L DMA Controller Driver
4 *
5 * Based on imx-dma.c
6 *
7 * Copyright (C) 2021 Renesas Electronics Corp.
8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
10 */
11
12 #include <linux/bitfield.h>
13 #include <linux/cleanup.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dmaengine.h>
16 #include <linux/interrupt.h>
17 #include <linux/iopoll.h>
18 #include <linux/irqchip/irq-renesas-rzv2h.h>
19 #include <linux/list.h>
20 #include <linux/module.h>
21 #include <linux/of.h>
22 #include <linux/of_dma.h>
23 #include <linux/of_platform.h>
24 #include <linux/platform_device.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/reset.h>
27 #include <linux/slab.h>
28 #include <linux/spinlock.h>
29
30 #include "../dmaengine.h"
31 #include "../virt-dma.h"
32
33 enum rz_dmac_prep_type {
34 RZ_DMAC_DESC_MEMCPY,
35 RZ_DMAC_DESC_SLAVE_SG,
36 };
37
38 struct rz_lmdesc {
39 u32 header;
40 u32 sa;
41 u32 da;
42 u32 tb;
43 u32 chcfg;
44 u32 chitvl;
45 u32 chext;
46 u32 nxla;
47 };
48
49 struct rz_dmac_desc {
50 struct virt_dma_desc vd;
51 dma_addr_t src;
52 dma_addr_t dest;
53 size_t len;
54 struct list_head node;
55 enum dma_transfer_direction direction;
56 enum rz_dmac_prep_type type;
57 /* For slave sg */
58 struct scatterlist *sg;
59 unsigned int sgcount;
60 };
61
62 #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd)
63
64 struct rz_dmac_chan {
65 struct virt_dma_chan vc;
66 void __iomem *ch_base;
67 void __iomem *ch_cmn_base;
68 unsigned int index;
69 struct rz_dmac_desc *desc;
70 int descs_allocated;
71
72 dma_addr_t src_per_address;
73 dma_addr_t dst_per_address;
74
75 u32 chcfg;
76 u32 chctrl;
77 int mid_rid;
78
79 struct list_head ld_free;
80 struct list_head ld_queue;
81 struct list_head ld_active;
82
83 struct {
84 struct rz_lmdesc *base;
85 struct rz_lmdesc *head;
86 struct rz_lmdesc *tail;
87 dma_addr_t base_dma;
88 } lmdesc;
89 };
90
91 #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan)
92
93 struct rz_dmac_icu {
94 struct platform_device *pdev;
95 u8 dmac_index;
96 };
97
98 struct rz_dmac {
99 struct dma_device engine;
100 struct rz_dmac_icu icu;
101 struct device *dev;
102 struct reset_control *rstc;
103 void __iomem *base;
104 void __iomem *ext_base;
105
106 unsigned int n_channels;
107 struct rz_dmac_chan *channels;
108
109 bool has_icu;
110
111 DECLARE_BITMAP(modules, 1024);
112 };
113
114 #define to_rz_dmac(d) container_of(d, struct rz_dmac, engine)
115
116 /*
117 * -----------------------------------------------------------------------------
118 * Registers
119 */
120
121 #define CHSTAT 0x0024
122 #define CHCTRL 0x0028
123 #define CHCFG 0x002c
124 #define NXLA 0x0038
125
126 #define DCTRL 0x0000
127
128 #define EACH_CHANNEL_OFFSET 0x0040
129 #define CHANNEL_0_7_OFFSET 0x0000
130 #define CHANNEL_0_7_COMMON_BASE 0x0300
131 #define CHANNEL_8_15_OFFSET 0x0400
132 #define CHANNEL_8_15_COMMON_BASE 0x0700
133
134 #define CHSTAT_ER BIT(4)
135 #define CHSTAT_EN BIT(0)
136
137 #define CHCTRL_CLRINTMSK BIT(17)
138 #define CHCTRL_CLRSUS BIT(9)
139 #define CHCTRL_CLRTC BIT(6)
140 #define CHCTRL_CLREND BIT(5)
141 #define CHCTRL_CLRRQ BIT(4)
142 #define CHCTRL_SWRST BIT(3)
143 #define CHCTRL_STG BIT(2)
144 #define CHCTRL_CLREN BIT(1)
145 #define CHCTRL_SETEN BIT(0)
146 #define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
147 CHCTRL_CLRTC | CHCTRL_CLREND | \
148 CHCTRL_CLRRQ | CHCTRL_SWRST | \
149 CHCTRL_CLREN)
150
151 #define CHCFG_DMS BIT(31)
152 #define CHCFG_DEM BIT(24)
153 #define CHCFG_DAD BIT(21)
154 #define CHCFG_SAD BIT(20)
155 #define CHCFG_REQD BIT(3)
156 #define CHCFG_SEL(bits) ((bits) & 0x07)
157 #define CHCFG_MEM_COPY (0x80400008)
158 #define CHCFG_FILL_DDS_MASK GENMASK(19, 16)
159 #define CHCFG_FILL_SDS_MASK GENMASK(15, 12)
160 #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22)
161 #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6)
162 #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5)
163 #define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5)
164
165 #define MID_RID_MASK GENMASK(9, 0)
166 #define CHCFG_MASK GENMASK(15, 10)
167 #define CHCFG_DS_INVALID 0xFF
168 #define DCTRL_LVINT BIT(1)
169 #define DCTRL_PR BIT(0)
170 #define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR)
171
172 /* LINK MODE DESCRIPTOR */
173 #define HEADER_LV BIT(0)
174
175 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16
176 #define RZ_DMAC_MAX_CHANNELS 16
177 #define DMAC_NR_LMDESC 64
178
179 /* RZ/V2H ICU related */
180 #define RZV2H_MAX_DMAC_INDEX 4
181
182 /*
183 * -----------------------------------------------------------------------------
184 * Device access
185 */
186
rz_dmac_writel(struct rz_dmac * dmac,unsigned int val,unsigned int offset)187 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
188 unsigned int offset)
189 {
190 writel(val, dmac->base + offset);
191 }
192
rz_dmac_ext_writel(struct rz_dmac * dmac,unsigned int val,unsigned int offset)193 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
194 unsigned int offset)
195 {
196 writel(val, dmac->ext_base + offset);
197 }
198
rz_dmac_ext_readl(struct rz_dmac * dmac,unsigned int offset)199 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
200 {
201 return readl(dmac->ext_base + offset);
202 }
203
rz_dmac_ch_writel(struct rz_dmac_chan * channel,unsigned int val,unsigned int offset,int which)204 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
205 unsigned int offset, int which)
206 {
207 if (which)
208 writel(val, channel->ch_base + offset);
209 else
210 writel(val, channel->ch_cmn_base + offset);
211 }
212
rz_dmac_ch_readl(struct rz_dmac_chan * channel,unsigned int offset,int which)213 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
214 unsigned int offset, int which)
215 {
216 if (which)
217 return readl(channel->ch_base + offset);
218 else
219 return readl(channel->ch_cmn_base + offset);
220 }
221
222 /*
223 * -----------------------------------------------------------------------------
224 * Initialization
225 */
226
rz_lmdesc_setup(struct rz_dmac_chan * channel,struct rz_lmdesc * lmdesc)227 static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
228 struct rz_lmdesc *lmdesc)
229 {
230 u32 nxla;
231
232 channel->lmdesc.base = lmdesc;
233 channel->lmdesc.head = lmdesc;
234 channel->lmdesc.tail = lmdesc;
235 nxla = channel->lmdesc.base_dma;
236 while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
237 lmdesc->header = 0;
238 nxla += sizeof(*lmdesc);
239 lmdesc->nxla = nxla;
240 lmdesc++;
241 }
242
243 lmdesc->header = 0;
244 lmdesc->nxla = channel->lmdesc.base_dma;
245 }
246
247 /*
248 * -----------------------------------------------------------------------------
249 * Descriptors preparation
250 */
251
rz_dmac_lmdesc_recycle(struct rz_dmac_chan * channel)252 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
253 {
254 struct rz_lmdesc *lmdesc = channel->lmdesc.head;
255
256 while (!(lmdesc->header & HEADER_LV)) {
257 lmdesc->header = 0;
258 lmdesc++;
259 if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
260 lmdesc = channel->lmdesc.base;
261 }
262 channel->lmdesc.head = lmdesc;
263 }
264
rz_dmac_enable_hw(struct rz_dmac_chan * channel)265 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
266 {
267 struct dma_chan *chan = &channel->vc.chan;
268 struct rz_dmac *dmac = to_rz_dmac(chan->device);
269 unsigned long flags;
270 u32 nxla;
271 u32 chctrl;
272 u32 chstat;
273
274 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
275
276 local_irq_save(flags);
277
278 rz_dmac_lmdesc_recycle(channel);
279
280 nxla = channel->lmdesc.base_dma +
281 (sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
282 channel->lmdesc.base));
283
284 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
285 if (!(chstat & CHSTAT_EN)) {
286 chctrl = (channel->chctrl | CHCTRL_SETEN);
287 rz_dmac_ch_writel(channel, nxla, NXLA, 1);
288 rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
289 rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
290 rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
291 }
292
293 local_irq_restore(flags);
294 }
295
rz_dmac_disable_hw(struct rz_dmac_chan * channel)296 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
297 {
298 struct dma_chan *chan = &channel->vc.chan;
299 struct rz_dmac *dmac = to_rz_dmac(chan->device);
300
301 dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
302
303 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
304 }
305
rz_dmac_set_dmars_register(struct rz_dmac * dmac,int nr,u32 dmars)306 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
307 {
308 u32 dmars_offset = (nr / 2) * 4;
309 u32 shift = (nr % 2) * 16;
310 u32 dmars32;
311
312 dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
313 dmars32 &= ~(0xffff << shift);
314 dmars32 |= dmars << shift;
315
316 rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
317 }
318
rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan * channel)319 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
320 {
321 struct dma_chan *chan = &channel->vc.chan;
322 struct rz_dmac *dmac = to_rz_dmac(chan->device);
323 struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
324 struct rz_dmac_desc *d = channel->desc;
325 u32 chcfg = CHCFG_MEM_COPY;
326
327 /* prepare descriptor */
328 lmdesc->sa = d->src;
329 lmdesc->da = d->dest;
330 lmdesc->tb = d->len;
331 lmdesc->chcfg = chcfg;
332 lmdesc->chitvl = 0;
333 lmdesc->chext = 0;
334 lmdesc->header = HEADER_LV;
335
336 if (dmac->has_icu) {
337 rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
338 channel->index,
339 RZV2H_ICU_DMAC_REQ_NO_DEFAULT);
340 } else {
341 rz_dmac_set_dmars_register(dmac, channel->index, 0);
342 }
343
344 channel->chcfg = chcfg;
345 channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
346 }
347
rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan * channel)348 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
349 {
350 struct dma_chan *chan = &channel->vc.chan;
351 struct rz_dmac *dmac = to_rz_dmac(chan->device);
352 struct rz_dmac_desc *d = channel->desc;
353 struct scatterlist *sg, *sgl = d->sg;
354 struct rz_lmdesc *lmdesc;
355 unsigned int i, sg_len = d->sgcount;
356
357 channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
358
359 if (d->direction == DMA_DEV_TO_MEM) {
360 channel->chcfg |= CHCFG_SAD;
361 channel->chcfg &= ~CHCFG_REQD;
362 } else {
363 channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
364 }
365
366 lmdesc = channel->lmdesc.tail;
367
368 for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
369 if (d->direction == DMA_DEV_TO_MEM) {
370 lmdesc->sa = channel->src_per_address;
371 lmdesc->da = sg_dma_address(sg);
372 } else {
373 lmdesc->sa = sg_dma_address(sg);
374 lmdesc->da = channel->dst_per_address;
375 }
376
377 lmdesc->tb = sg_dma_len(sg);
378 lmdesc->chitvl = 0;
379 lmdesc->chext = 0;
380 if (i == (sg_len - 1)) {
381 lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
382 lmdesc->header = HEADER_LV;
383 } else {
384 lmdesc->chcfg = channel->chcfg;
385 lmdesc->header = HEADER_LV;
386 }
387 if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
388 lmdesc = channel->lmdesc.base;
389 }
390
391 channel->lmdesc.tail = lmdesc;
392
393 if (dmac->has_icu) {
394 rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
395 channel->index, channel->mid_rid);
396 } else {
397 rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
398 }
399
400 channel->chctrl = CHCTRL_SETEN;
401 }
402
rz_dmac_xfer_desc(struct rz_dmac_chan * chan)403 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
404 {
405 struct rz_dmac_desc *d = chan->desc;
406 struct virt_dma_desc *vd;
407
408 vd = vchan_next_desc(&chan->vc);
409 if (!vd)
410 return 0;
411
412 list_del(&vd->node);
413
414 switch (d->type) {
415 case RZ_DMAC_DESC_MEMCPY:
416 rz_dmac_prepare_desc_for_memcpy(chan);
417 break;
418
419 case RZ_DMAC_DESC_SLAVE_SG:
420 rz_dmac_prepare_descs_for_slave_sg(chan);
421 break;
422
423 default:
424 return -EINVAL;
425 }
426
427 rz_dmac_enable_hw(chan);
428
429 return 0;
430 }
431
432 /*
433 * -----------------------------------------------------------------------------
434 * DMA engine operations
435 */
436
rz_dmac_alloc_chan_resources(struct dma_chan * chan)437 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
438 {
439 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
440
441 while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
442 struct rz_dmac_desc *desc;
443
444 desc = kzalloc_obj(*desc);
445 if (!desc)
446 break;
447
448 /* No need to lock. This is called only for the 1st client. */
449 list_add_tail(&desc->node, &channel->ld_free);
450 channel->descs_allocated++;
451 }
452
453 if (!channel->descs_allocated)
454 return -ENOMEM;
455
456 return channel->descs_allocated;
457 }
458
rz_dmac_free_chan_resources(struct dma_chan * chan)459 static void rz_dmac_free_chan_resources(struct dma_chan *chan)
460 {
461 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
462 struct rz_dmac *dmac = to_rz_dmac(chan->device);
463 struct rz_lmdesc *lmdesc = channel->lmdesc.base;
464 struct rz_dmac_desc *desc, *_desc;
465 unsigned long flags;
466 unsigned int i;
467
468 spin_lock_irqsave(&channel->vc.lock, flags);
469
470 for (i = 0; i < DMAC_NR_LMDESC; i++)
471 lmdesc[i].header = 0;
472
473 rz_dmac_disable_hw(channel);
474 list_splice_tail_init(&channel->ld_active, &channel->ld_free);
475 list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
476
477 if (channel->mid_rid >= 0) {
478 clear_bit(channel->mid_rid, dmac->modules);
479 channel->mid_rid = -EINVAL;
480 }
481
482 spin_unlock_irqrestore(&channel->vc.lock, flags);
483
484 list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
485 kfree(desc);
486 channel->descs_allocated--;
487 }
488
489 INIT_LIST_HEAD(&channel->ld_free);
490 vchan_free_chan_resources(&channel->vc);
491 }
492
493 static struct dma_async_tx_descriptor *
rz_dmac_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)494 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
495 size_t len, unsigned long flags)
496 {
497 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
498 struct rz_dmac *dmac = to_rz_dmac(chan->device);
499 struct rz_dmac_desc *desc;
500
501 dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
502 __func__, channel->index, &src, &dest, len);
503
504 scoped_guard(spinlock_irqsave, &channel->vc.lock) {
505 if (list_empty(&channel->ld_free))
506 return NULL;
507
508 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
509
510 desc->type = RZ_DMAC_DESC_MEMCPY;
511 desc->src = src;
512 desc->dest = dest;
513 desc->len = len;
514 desc->direction = DMA_MEM_TO_MEM;
515
516 list_move_tail(channel->ld_free.next, &channel->ld_queue);
517 }
518
519 return vchan_tx_prep(&channel->vc, &desc->vd, flags);
520 }
521
522 static struct dma_async_tx_descriptor *
rz_dmac_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)523 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
524 unsigned int sg_len,
525 enum dma_transfer_direction direction,
526 unsigned long flags, void *context)
527 {
528 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
529 struct rz_dmac_desc *desc;
530 struct scatterlist *sg;
531 int dma_length = 0;
532 int i = 0;
533
534 scoped_guard(spinlock_irqsave, &channel->vc.lock) {
535 if (list_empty(&channel->ld_free))
536 return NULL;
537
538 desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
539
540 for_each_sg(sgl, sg, sg_len, i)
541 dma_length += sg_dma_len(sg);
542
543 desc->type = RZ_DMAC_DESC_SLAVE_SG;
544 desc->sg = sgl;
545 desc->sgcount = sg_len;
546 desc->len = dma_length;
547 desc->direction = direction;
548
549 if (direction == DMA_DEV_TO_MEM)
550 desc->src = channel->src_per_address;
551 else
552 desc->dest = channel->dst_per_address;
553
554 list_move_tail(channel->ld_free.next, &channel->ld_queue);
555 }
556
557 return vchan_tx_prep(&channel->vc, &desc->vd, flags);
558 }
559
rz_dmac_terminate_all(struct dma_chan * chan)560 static int rz_dmac_terminate_all(struct dma_chan *chan)
561 {
562 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
563 struct rz_lmdesc *lmdesc = channel->lmdesc.base;
564 unsigned long flags;
565 unsigned int i;
566 LIST_HEAD(head);
567
568 spin_lock_irqsave(&channel->vc.lock, flags);
569 rz_dmac_disable_hw(channel);
570 for (i = 0; i < DMAC_NR_LMDESC; i++)
571 lmdesc[i].header = 0;
572
573 list_splice_tail_init(&channel->ld_active, &channel->ld_free);
574 list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
575 vchan_get_all_descriptors(&channel->vc, &head);
576 spin_unlock_irqrestore(&channel->vc.lock, flags);
577 vchan_dma_desc_free_list(&channel->vc, &head);
578
579 return 0;
580 }
581
rz_dmac_issue_pending(struct dma_chan * chan)582 static void rz_dmac_issue_pending(struct dma_chan *chan)
583 {
584 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
585 struct rz_dmac *dmac = to_rz_dmac(chan->device);
586 struct rz_dmac_desc *desc;
587 unsigned long flags;
588
589 spin_lock_irqsave(&channel->vc.lock, flags);
590
591 if (!list_empty(&channel->ld_queue)) {
592 desc = list_first_entry(&channel->ld_queue,
593 struct rz_dmac_desc, node);
594 channel->desc = desc;
595 if (vchan_issue_pending(&channel->vc)) {
596 if (rz_dmac_xfer_desc(channel) < 0)
597 dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
598 channel->index);
599 else
600 list_move_tail(channel->ld_queue.next,
601 &channel->ld_active);
602 }
603 }
604
605 spin_unlock_irqrestore(&channel->vc.lock, flags);
606 }
607
rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)608 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
609 {
610 u8 i;
611 static const enum dma_slave_buswidth ds_lut[] = {
612 DMA_SLAVE_BUSWIDTH_1_BYTE,
613 DMA_SLAVE_BUSWIDTH_2_BYTES,
614 DMA_SLAVE_BUSWIDTH_4_BYTES,
615 DMA_SLAVE_BUSWIDTH_8_BYTES,
616 DMA_SLAVE_BUSWIDTH_16_BYTES,
617 DMA_SLAVE_BUSWIDTH_32_BYTES,
618 DMA_SLAVE_BUSWIDTH_64_BYTES,
619 DMA_SLAVE_BUSWIDTH_128_BYTES,
620 };
621
622 for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
623 if (ds_lut[i] == ds)
624 return i;
625 }
626
627 return CHCFG_DS_INVALID;
628 }
629
rz_dmac_config(struct dma_chan * chan,struct dma_slave_config * config)630 static int rz_dmac_config(struct dma_chan *chan,
631 struct dma_slave_config *config)
632 {
633 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
634 u32 val;
635
636 channel->dst_per_address = config->dst_addr;
637 channel->chcfg &= ~CHCFG_FILL_DDS_MASK;
638 if (channel->dst_per_address) {
639 val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
640 if (val == CHCFG_DS_INVALID)
641 return -EINVAL;
642
643 channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val);
644 }
645
646 channel->src_per_address = config->src_addr;
647 channel->chcfg &= ~CHCFG_FILL_SDS_MASK;
648 if (channel->src_per_address) {
649 val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
650 if (val == CHCFG_DS_INVALID)
651 return -EINVAL;
652
653 channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val);
654 }
655
656 return 0;
657 }
658
rz_dmac_virt_desc_free(struct virt_dma_desc * vd)659 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
660 {
661 /*
662 * Place holder
663 * Descriptor allocation is done during alloc_chan_resources and
664 * get freed during free_chan_resources.
665 * list is used to manage the descriptors and avoid any memory
666 * allocation/free during DMA read/write.
667 */
668 }
669
rz_dmac_device_synchronize(struct dma_chan * chan)670 static void rz_dmac_device_synchronize(struct dma_chan *chan)
671 {
672 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
673 struct rz_dmac *dmac = to_rz_dmac(chan->device);
674 u32 chstat;
675 int ret;
676
677 ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
678 100, 100000, false, channel, CHSTAT, 1);
679 if (ret < 0)
680 dev_warn(dmac->dev, "DMA Timeout");
681
682 if (dmac->has_icu) {
683 rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
684 channel->index,
685 RZV2H_ICU_DMAC_REQ_NO_DEFAULT);
686 } else {
687 rz_dmac_set_dmars_register(dmac, channel->index, 0);
688 }
689 }
690
691 /*
692 * -----------------------------------------------------------------------------
693 * IRQ handling
694 */
695
rz_dmac_irq_handle_channel(struct rz_dmac_chan * channel)696 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
697 {
698 struct dma_chan *chan = &channel->vc.chan;
699 struct rz_dmac *dmac = to_rz_dmac(chan->device);
700 u32 chstat, chctrl;
701
702 chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
703 if (chstat & CHSTAT_ER) {
704 dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
705 channel->index, chstat);
706
707 scoped_guard(spinlock_irqsave, &channel->vc.lock)
708 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
709 goto done;
710 }
711
712 chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
713 rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
714 done:
715 return;
716 }
717
rz_dmac_irq_handler(int irq,void * dev_id)718 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
719 {
720 struct rz_dmac_chan *channel = dev_id;
721
722 if (channel) {
723 rz_dmac_irq_handle_channel(channel);
724 return IRQ_WAKE_THREAD;
725 }
726 /* handle DMAERR irq */
727 return IRQ_HANDLED;
728 }
729
rz_dmac_irq_handler_thread(int irq,void * dev_id)730 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
731 {
732 struct rz_dmac_chan *channel = dev_id;
733 struct rz_dmac_desc *desc = NULL;
734 unsigned long flags;
735
736 spin_lock_irqsave(&channel->vc.lock, flags);
737
738 if (list_empty(&channel->ld_active)) {
739 /* Someone might have called terminate all */
740 goto out;
741 }
742
743 desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
744 vchan_cookie_complete(&desc->vd);
745 list_move_tail(channel->ld_active.next, &channel->ld_free);
746 if (!list_empty(&channel->ld_queue)) {
747 desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
748 node);
749 channel->desc = desc;
750 if (rz_dmac_xfer_desc(channel) == 0)
751 list_move_tail(channel->ld_queue.next, &channel->ld_active);
752 }
753 out:
754 spin_unlock_irqrestore(&channel->vc.lock, flags);
755
756 return IRQ_HANDLED;
757 }
758
759 /*
760 * -----------------------------------------------------------------------------
761 * OF xlate and channel filter
762 */
763
rz_dmac_chan_filter(struct dma_chan * chan,void * arg)764 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
765 {
766 struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
767 struct rz_dmac *dmac = to_rz_dmac(chan->device);
768 struct of_phandle_args *dma_spec = arg;
769 u32 ch_cfg;
770
771 channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
772 ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
773 channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
774 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
775
776 return !test_and_set_bit(channel->mid_rid, dmac->modules);
777 }
778
rz_dmac_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)779 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
780 struct of_dma *ofdma)
781 {
782 dma_cap_mask_t mask;
783
784 if (dma_spec->args_count != 1)
785 return NULL;
786
787 /* Only slave DMA channels can be allocated via DT */
788 dma_cap_zero(mask);
789 dma_cap_set(DMA_SLAVE, mask);
790
791 return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec,
792 ofdma->of_node);
793 }
794
795 /*
796 * -----------------------------------------------------------------------------
797 * Probe and remove
798 */
799
rz_dmac_chan_probe(struct rz_dmac * dmac,struct rz_dmac_chan * channel,u8 index)800 static int rz_dmac_chan_probe(struct rz_dmac *dmac,
801 struct rz_dmac_chan *channel,
802 u8 index)
803 {
804 struct platform_device *pdev = to_platform_device(dmac->dev);
805 struct rz_lmdesc *lmdesc;
806 char pdev_irqname[6];
807 char *irqname;
808 int irq, ret;
809
810 channel->index = index;
811 channel->mid_rid = -EINVAL;
812
813 /* Request the channel interrupt. */
814 scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
815 irq = platform_get_irq_byname(pdev, pdev_irqname);
816 if (irq < 0)
817 return irq;
818
819 irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
820 dev_name(dmac->dev), index);
821 if (!irqname)
822 return -ENOMEM;
823
824 ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler,
825 rz_dmac_irq_handler_thread, 0,
826 irqname, channel);
827 if (ret) {
828 dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret);
829 return ret;
830 }
831
832 /* Set io base address for each channel */
833 if (index < 8) {
834 channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
835 EACH_CHANNEL_OFFSET * index;
836 channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
837 } else {
838 channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
839 EACH_CHANNEL_OFFSET * (index - 8);
840 channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
841 }
842
843 /* Allocate descriptors */
844 lmdesc = dma_alloc_coherent(&pdev->dev,
845 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
846 &channel->lmdesc.base_dma, GFP_KERNEL);
847 if (!lmdesc) {
848 dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
849 return -ENOMEM;
850 }
851 rz_lmdesc_setup(channel, lmdesc);
852
853 /* Initialize register for each channel */
854 rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
855
856 channel->vc.desc_free = rz_dmac_virt_desc_free;
857 vchan_init(&channel->vc, &dmac->engine);
858 INIT_LIST_HEAD(&channel->ld_queue);
859 INIT_LIST_HEAD(&channel->ld_free);
860 INIT_LIST_HEAD(&channel->ld_active);
861
862 return 0;
863 }
864
rz_dmac_put_device(void * _dev)865 static void rz_dmac_put_device(void *_dev)
866 {
867 struct device *dev = _dev;
868
869 put_device(dev);
870 }
871
rz_dmac_parse_of_icu(struct device * dev,struct rz_dmac * dmac)872 static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac)
873 {
874 struct device_node *np = dev->of_node;
875 struct of_phandle_args args;
876 uint32_t dmac_index;
877 int ret;
878
879 ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args);
880 if (ret == -ENOENT)
881 return 0;
882 if (ret)
883 return ret;
884
885 dmac->has_icu = true;
886
887 dmac->icu.pdev = of_find_device_by_node(args.np);
888 of_node_put(args.np);
889 if (!dmac->icu.pdev) {
890 dev_err(dev, "ICU device not found.\n");
891 return -ENODEV;
892 }
893
894 ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev);
895 if (ret)
896 return ret;
897
898 dmac_index = args.args[0];
899 if (dmac_index > RZV2H_MAX_DMAC_INDEX) {
900 dev_err(dev, "DMAC index %u invalid.\n", dmac_index);
901 return -EINVAL;
902 }
903 dmac->icu.dmac_index = dmac_index;
904
905 return 0;
906 }
907
rz_dmac_parse_of(struct device * dev,struct rz_dmac * dmac)908 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
909 {
910 struct device_node *np = dev->of_node;
911 int ret;
912
913 ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
914 if (ret < 0) {
915 dev_err(dev, "unable to read dma-channels property\n");
916 return ret;
917 }
918
919 if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
920 dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
921 return -EINVAL;
922 }
923
924 return rz_dmac_parse_of_icu(dev, dmac);
925 }
926
rz_dmac_probe(struct platform_device * pdev)927 static int rz_dmac_probe(struct platform_device *pdev)
928 {
929 const char *irqname = "error";
930 struct dma_device *engine;
931 struct rz_dmac *dmac;
932 int channel_num;
933 int ret;
934 int irq;
935 u8 i;
936
937 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
938 if (!dmac)
939 return -ENOMEM;
940
941 dmac->dev = &pdev->dev;
942 platform_set_drvdata(pdev, dmac);
943
944 ret = rz_dmac_parse_of(&pdev->dev, dmac);
945 if (ret < 0)
946 return ret;
947
948 dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
949 sizeof(*dmac->channels), GFP_KERNEL);
950 if (!dmac->channels)
951 return -ENOMEM;
952
953 /* Request resources */
954 dmac->base = devm_platform_ioremap_resource(pdev, 0);
955 if (IS_ERR(dmac->base))
956 return PTR_ERR(dmac->base);
957
958 if (!dmac->has_icu) {
959 dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
960 if (IS_ERR(dmac->ext_base))
961 return PTR_ERR(dmac->ext_base);
962 }
963
964 /* Register interrupt handler for error */
965 irq = platform_get_irq_byname(pdev, irqname);
966 if (irq < 0)
967 return irq;
968
969 ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
970 irqname, NULL);
971 if (ret) {
972 dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
973 irq, ret);
974 return ret;
975 }
976
977 /* Initialize the channels. */
978 INIT_LIST_HEAD(&dmac->engine.channels);
979
980 dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev);
981 if (IS_ERR(dmac->rstc))
982 return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),
983 "failed to get resets\n");
984
985 pm_runtime_enable(&pdev->dev);
986 ret = pm_runtime_resume_and_get(&pdev->dev);
987 if (ret < 0) {
988 dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
989 goto err_pm_disable;
990 }
991
992 ret = reset_control_deassert(dmac->rstc);
993 if (ret)
994 goto err_pm_runtime_put;
995
996 for (i = 0; i < dmac->n_channels; i++) {
997 ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
998 if (ret < 0)
999 goto err;
1000 }
1001
1002 /* Register the DMAC as a DMA provider for DT. */
1003 ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
1004 NULL);
1005 if (ret < 0)
1006 goto err;
1007
1008 /* Register the DMA engine device. */
1009 engine = &dmac->engine;
1010 dma_cap_set(DMA_SLAVE, engine->cap_mask);
1011 dma_cap_set(DMA_MEMCPY, engine->cap_mask);
1012 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
1013 rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
1014
1015 engine->dev = &pdev->dev;
1016
1017 engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
1018 engine->device_free_chan_resources = rz_dmac_free_chan_resources;
1019 engine->device_tx_status = dma_cookie_status;
1020 engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
1021 engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
1022 engine->device_config = rz_dmac_config;
1023 engine->device_terminate_all = rz_dmac_terminate_all;
1024 engine->device_issue_pending = rz_dmac_issue_pending;
1025 engine->device_synchronize = rz_dmac_device_synchronize;
1026
1027 engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
1028 dma_set_max_seg_size(engine->dev, U32_MAX);
1029
1030 ret = dma_async_device_register(engine);
1031 if (ret < 0) {
1032 dev_err(&pdev->dev, "unable to register\n");
1033 goto dma_register_err;
1034 }
1035 return 0;
1036
1037 dma_register_err:
1038 of_dma_controller_free(pdev->dev.of_node);
1039 err:
1040 channel_num = i ? i - 1 : 0;
1041 for (i = 0; i < channel_num; i++) {
1042 struct rz_dmac_chan *channel = &dmac->channels[i];
1043
1044 dma_free_coherent(&pdev->dev,
1045 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
1046 channel->lmdesc.base,
1047 channel->lmdesc.base_dma);
1048 }
1049
1050 reset_control_assert(dmac->rstc);
1051 err_pm_runtime_put:
1052 pm_runtime_put(&pdev->dev);
1053 err_pm_disable:
1054 pm_runtime_disable(&pdev->dev);
1055
1056 return ret;
1057 }
1058
rz_dmac_remove(struct platform_device * pdev)1059 static void rz_dmac_remove(struct platform_device *pdev)
1060 {
1061 struct rz_dmac *dmac = platform_get_drvdata(pdev);
1062 unsigned int i;
1063
1064 dma_async_device_unregister(&dmac->engine);
1065 of_dma_controller_free(pdev->dev.of_node);
1066 for (i = 0; i < dmac->n_channels; i++) {
1067 struct rz_dmac_chan *channel = &dmac->channels[i];
1068
1069 dma_free_coherent(&pdev->dev,
1070 sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
1071 channel->lmdesc.base,
1072 channel->lmdesc.base_dma);
1073 }
1074 reset_control_assert(dmac->rstc);
1075 pm_runtime_put(&pdev->dev);
1076 pm_runtime_disable(&pdev->dev);
1077 }
1078
1079 static const struct of_device_id of_rz_dmac_match[] = {
1080 { .compatible = "renesas,r9a09g057-dmac", },
1081 { .compatible = "renesas,rz-dmac", },
1082 { /* Sentinel */ }
1083 };
1084 MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
1085
1086 static struct platform_driver rz_dmac_driver = {
1087 .driver = {
1088 .name = "rz-dmac",
1089 .of_match_table = of_rz_dmac_match,
1090 },
1091 .probe = rz_dmac_probe,
1092 .remove = rz_dmac_remove,
1093 };
1094
1095 module_platform_driver(rz_dmac_driver);
1096
1097 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
1098 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
1099 MODULE_LICENSE("GPL v2");
1100