xref: /linux/drivers/dma/sh/rz-dmac.c (revision e8529dcb1218ce176d5e84168568f69e1d9de109)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Renesas RZ/G2L DMA Controller Driver
4  *
5  * Based on imx-dma.c
6  *
7  * Copyright (C) 2021 Renesas Electronics Corp.
8  * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
9  * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
10  */
11 
12 #include <linux/bitfield.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/dmaengine.h>
15 #include <linux/interrupt.h>
16 #include <linux/iopoll.h>
17 #include <linux/list.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/of_dma.h>
21 #include <linux/of_platform.h>
22 #include <linux/platform_device.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/reset.h>
25 #include <linux/slab.h>
26 #include <linux/spinlock.h>
27 
28 #include "../dmaengine.h"
29 #include "../virt-dma.h"
30 
31 enum  rz_dmac_prep_type {
32 	RZ_DMAC_DESC_MEMCPY,
33 	RZ_DMAC_DESC_SLAVE_SG,
34 };
35 
36 struct rz_lmdesc {
37 	u32 header;
38 	u32 sa;
39 	u32 da;
40 	u32 tb;
41 	u32 chcfg;
42 	u32 chitvl;
43 	u32 chext;
44 	u32 nxla;
45 };
46 
47 struct rz_dmac_desc {
48 	struct virt_dma_desc vd;
49 	dma_addr_t src;
50 	dma_addr_t dest;
51 	size_t len;
52 	struct list_head node;
53 	enum dma_transfer_direction direction;
54 	enum rz_dmac_prep_type type;
55 	/* For slave sg */
56 	struct scatterlist *sg;
57 	unsigned int sgcount;
58 };
59 
60 #define to_rz_dmac_desc(d)	container_of(d, struct rz_dmac_desc, vd)
61 
62 struct rz_dmac_chan {
63 	struct virt_dma_chan vc;
64 	void __iomem *ch_base;
65 	void __iomem *ch_cmn_base;
66 	unsigned int index;
67 	int irq;
68 	struct rz_dmac_desc *desc;
69 	int descs_allocated;
70 
71 	dma_addr_t src_per_address;
72 	dma_addr_t dst_per_address;
73 
74 	u32 chcfg;
75 	u32 chctrl;
76 	int mid_rid;
77 
78 	struct list_head ld_free;
79 	struct list_head ld_queue;
80 	struct list_head ld_active;
81 
82 	struct {
83 		struct rz_lmdesc *base;
84 		struct rz_lmdesc *head;
85 		struct rz_lmdesc *tail;
86 		dma_addr_t base_dma;
87 	} lmdesc;
88 };
89 
90 #define to_rz_dmac_chan(c)	container_of(c, struct rz_dmac_chan, vc.chan)
91 
92 struct rz_dmac {
93 	struct dma_device engine;
94 	struct device *dev;
95 	struct reset_control *rstc;
96 	void __iomem *base;
97 	void __iomem *ext_base;
98 
99 	unsigned int n_channels;
100 	struct rz_dmac_chan *channels;
101 
102 	DECLARE_BITMAP(modules, 1024);
103 };
104 
105 #define to_rz_dmac(d)	container_of(d, struct rz_dmac, engine)
106 
107 /*
108  * -----------------------------------------------------------------------------
109  * Registers
110  */
111 
112 #define CHSTAT				0x0024
113 #define CHCTRL				0x0028
114 #define CHCFG				0x002c
115 #define NXLA				0x0038
116 
117 #define DCTRL				0x0000
118 
119 #define EACH_CHANNEL_OFFSET		0x0040
120 #define CHANNEL_0_7_OFFSET		0x0000
121 #define CHANNEL_0_7_COMMON_BASE		0x0300
122 #define CHANNEL_8_15_OFFSET		0x0400
123 #define CHANNEL_8_15_COMMON_BASE	0x0700
124 
125 #define CHSTAT_ER			BIT(4)
126 #define CHSTAT_EN			BIT(0)
127 
128 #define CHCTRL_CLRINTMSK		BIT(17)
129 #define CHCTRL_CLRSUS			BIT(9)
130 #define CHCTRL_CLRTC			BIT(6)
131 #define CHCTRL_CLREND			BIT(5)
132 #define CHCTRL_CLRRQ			BIT(4)
133 #define CHCTRL_SWRST			BIT(3)
134 #define CHCTRL_STG			BIT(2)
135 #define CHCTRL_CLREN			BIT(1)
136 #define CHCTRL_SETEN			BIT(0)
137 #define CHCTRL_DEFAULT			(CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
138 					 CHCTRL_CLRTC |	CHCTRL_CLREND | \
139 					 CHCTRL_CLRRQ | CHCTRL_SWRST | \
140 					 CHCTRL_CLREN)
141 
142 #define CHCFG_DMS			BIT(31)
143 #define CHCFG_DEM			BIT(24)
144 #define CHCFG_DAD			BIT(21)
145 #define CHCFG_SAD			BIT(20)
146 #define CHCFG_REQD			BIT(3)
147 #define CHCFG_SEL(bits)			((bits) & 0x07)
148 #define CHCFG_MEM_COPY			(0x80400008)
149 #define CHCFG_FILL_DDS_MASK		GENMASK(19, 16)
150 #define CHCFG_FILL_SDS_MASK		GENMASK(15, 12)
151 #define CHCFG_FILL_TM(a)		(((a) & BIT(5)) << 22)
152 #define CHCFG_FILL_AM(a)		(((a) & GENMASK(4, 2)) << 6)
153 #define CHCFG_FILL_LVL(a)		(((a) & BIT(1)) << 5)
154 #define CHCFG_FILL_HIEN(a)		(((a) & BIT(0)) << 5)
155 
156 #define MID_RID_MASK			GENMASK(9, 0)
157 #define CHCFG_MASK			GENMASK(15, 10)
158 #define CHCFG_DS_INVALID		0xFF
159 #define DCTRL_LVINT			BIT(1)
160 #define DCTRL_PR			BIT(0)
161 #define DCTRL_DEFAULT			(DCTRL_LVINT | DCTRL_PR)
162 
163 /* LINK MODE DESCRIPTOR */
164 #define HEADER_LV			BIT(0)
165 
166 #define RZ_DMAC_MAX_CHAN_DESCRIPTORS	16
167 #define RZ_DMAC_MAX_CHANNELS		16
168 #define DMAC_NR_LMDESC			64
169 
170 /*
171  * -----------------------------------------------------------------------------
172  * Device access
173  */
174 
rz_dmac_writel(struct rz_dmac * dmac,unsigned int val,unsigned int offset)175 static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
176 			   unsigned int offset)
177 {
178 	writel(val, dmac->base + offset);
179 }
180 
rz_dmac_ext_writel(struct rz_dmac * dmac,unsigned int val,unsigned int offset)181 static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
182 			       unsigned int offset)
183 {
184 	writel(val, dmac->ext_base + offset);
185 }
186 
rz_dmac_ext_readl(struct rz_dmac * dmac,unsigned int offset)187 static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
188 {
189 	return readl(dmac->ext_base + offset);
190 }
191 
rz_dmac_ch_writel(struct rz_dmac_chan * channel,unsigned int val,unsigned int offset,int which)192 static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
193 			      unsigned int offset, int which)
194 {
195 	if (which)
196 		writel(val, channel->ch_base + offset);
197 	else
198 		writel(val, channel->ch_cmn_base + offset);
199 }
200 
rz_dmac_ch_readl(struct rz_dmac_chan * channel,unsigned int offset,int which)201 static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
202 			    unsigned int offset, int which)
203 {
204 	if (which)
205 		return readl(channel->ch_base + offset);
206 	else
207 		return readl(channel->ch_cmn_base + offset);
208 }
209 
210 /*
211  * -----------------------------------------------------------------------------
212  * Initialization
213  */
214 
rz_lmdesc_setup(struct rz_dmac_chan * channel,struct rz_lmdesc * lmdesc)215 static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
216 			    struct rz_lmdesc *lmdesc)
217 {
218 	u32 nxla;
219 
220 	channel->lmdesc.base = lmdesc;
221 	channel->lmdesc.head = lmdesc;
222 	channel->lmdesc.tail = lmdesc;
223 	nxla = channel->lmdesc.base_dma;
224 	while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
225 		lmdesc->header = 0;
226 		nxla += sizeof(*lmdesc);
227 		lmdesc->nxla = nxla;
228 		lmdesc++;
229 	}
230 
231 	lmdesc->header = 0;
232 	lmdesc->nxla = channel->lmdesc.base_dma;
233 }
234 
235 /*
236  * -----------------------------------------------------------------------------
237  * Descriptors preparation
238  */
239 
rz_dmac_lmdesc_recycle(struct rz_dmac_chan * channel)240 static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
241 {
242 	struct rz_lmdesc *lmdesc = channel->lmdesc.head;
243 
244 	while (!(lmdesc->header & HEADER_LV)) {
245 		lmdesc->header = 0;
246 		lmdesc++;
247 		if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
248 			lmdesc = channel->lmdesc.base;
249 	}
250 	channel->lmdesc.head = lmdesc;
251 }
252 
rz_dmac_enable_hw(struct rz_dmac_chan * channel)253 static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
254 {
255 	struct dma_chan *chan = &channel->vc.chan;
256 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
257 	unsigned long flags;
258 	u32 nxla;
259 	u32 chctrl;
260 	u32 chstat;
261 
262 	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
263 
264 	local_irq_save(flags);
265 
266 	rz_dmac_lmdesc_recycle(channel);
267 
268 	nxla = channel->lmdesc.base_dma +
269 		(sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
270 					     channel->lmdesc.base));
271 
272 	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
273 	if (!(chstat & CHSTAT_EN)) {
274 		chctrl = (channel->chctrl | CHCTRL_SETEN);
275 		rz_dmac_ch_writel(channel, nxla, NXLA, 1);
276 		rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
277 		rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
278 		rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
279 	}
280 
281 	local_irq_restore(flags);
282 }
283 
rz_dmac_disable_hw(struct rz_dmac_chan * channel)284 static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
285 {
286 	struct dma_chan *chan = &channel->vc.chan;
287 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
288 	unsigned long flags;
289 
290 	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
291 
292 	local_irq_save(flags);
293 	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
294 	local_irq_restore(flags);
295 }
296 
rz_dmac_set_dmars_register(struct rz_dmac * dmac,int nr,u32 dmars)297 static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
298 {
299 	u32 dmars_offset = (nr / 2) * 4;
300 	u32 shift = (nr % 2) * 16;
301 	u32 dmars32;
302 
303 	dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
304 	dmars32 &= ~(0xffff << shift);
305 	dmars32 |= dmars << shift;
306 
307 	rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
308 }
309 
rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan * channel)310 static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
311 {
312 	struct dma_chan *chan = &channel->vc.chan;
313 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
314 	struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
315 	struct rz_dmac_desc *d = channel->desc;
316 	u32 chcfg = CHCFG_MEM_COPY;
317 
318 	/* prepare descriptor */
319 	lmdesc->sa = d->src;
320 	lmdesc->da = d->dest;
321 	lmdesc->tb = d->len;
322 	lmdesc->chcfg = chcfg;
323 	lmdesc->chitvl = 0;
324 	lmdesc->chext = 0;
325 	lmdesc->header = HEADER_LV;
326 
327 	rz_dmac_set_dmars_register(dmac, channel->index, 0);
328 
329 	channel->chcfg = chcfg;
330 	channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
331 }
332 
rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan * channel)333 static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
334 {
335 	struct dma_chan *chan = &channel->vc.chan;
336 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
337 	struct rz_dmac_desc *d = channel->desc;
338 	struct scatterlist *sg, *sgl = d->sg;
339 	struct rz_lmdesc *lmdesc;
340 	unsigned int i, sg_len = d->sgcount;
341 
342 	channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
343 
344 	if (d->direction == DMA_DEV_TO_MEM) {
345 		channel->chcfg |= CHCFG_SAD;
346 		channel->chcfg &= ~CHCFG_REQD;
347 	} else {
348 		channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
349 	}
350 
351 	lmdesc = channel->lmdesc.tail;
352 
353 	for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
354 		if (d->direction == DMA_DEV_TO_MEM) {
355 			lmdesc->sa = channel->src_per_address;
356 			lmdesc->da = sg_dma_address(sg);
357 		} else {
358 			lmdesc->sa = sg_dma_address(sg);
359 			lmdesc->da = channel->dst_per_address;
360 		}
361 
362 		lmdesc->tb = sg_dma_len(sg);
363 		lmdesc->chitvl = 0;
364 		lmdesc->chext = 0;
365 		if (i == (sg_len - 1)) {
366 			lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
367 			lmdesc->header = HEADER_LV;
368 		} else {
369 			lmdesc->chcfg = channel->chcfg;
370 			lmdesc->header = HEADER_LV;
371 		}
372 		if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
373 			lmdesc = channel->lmdesc.base;
374 	}
375 
376 	channel->lmdesc.tail = lmdesc;
377 
378 	rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
379 	channel->chctrl = CHCTRL_SETEN;
380 }
381 
rz_dmac_xfer_desc(struct rz_dmac_chan * chan)382 static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
383 {
384 	struct rz_dmac_desc *d = chan->desc;
385 	struct virt_dma_desc *vd;
386 
387 	vd = vchan_next_desc(&chan->vc);
388 	if (!vd)
389 		return 0;
390 
391 	list_del(&vd->node);
392 
393 	switch (d->type) {
394 	case RZ_DMAC_DESC_MEMCPY:
395 		rz_dmac_prepare_desc_for_memcpy(chan);
396 		break;
397 
398 	case RZ_DMAC_DESC_SLAVE_SG:
399 		rz_dmac_prepare_descs_for_slave_sg(chan);
400 		break;
401 
402 	default:
403 		return -EINVAL;
404 	}
405 
406 	rz_dmac_enable_hw(chan);
407 
408 	return 0;
409 }
410 
411 /*
412  * -----------------------------------------------------------------------------
413  * DMA engine operations
414  */
415 
rz_dmac_alloc_chan_resources(struct dma_chan * chan)416 static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
417 {
418 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
419 
420 	while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
421 		struct rz_dmac_desc *desc;
422 
423 		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
424 		if (!desc)
425 			break;
426 
427 		list_add_tail(&desc->node, &channel->ld_free);
428 		channel->descs_allocated++;
429 	}
430 
431 	if (!channel->descs_allocated)
432 		return -ENOMEM;
433 
434 	return channel->descs_allocated;
435 }
436 
rz_dmac_free_chan_resources(struct dma_chan * chan)437 static void rz_dmac_free_chan_resources(struct dma_chan *chan)
438 {
439 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
440 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
441 	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
442 	struct rz_dmac_desc *desc, *_desc;
443 	unsigned long flags;
444 	unsigned int i;
445 
446 	spin_lock_irqsave(&channel->vc.lock, flags);
447 
448 	for (i = 0; i < DMAC_NR_LMDESC; i++)
449 		lmdesc[i].header = 0;
450 
451 	rz_dmac_disable_hw(channel);
452 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
453 	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
454 
455 	if (channel->mid_rid >= 0) {
456 		clear_bit(channel->mid_rid, dmac->modules);
457 		channel->mid_rid = -EINVAL;
458 	}
459 
460 	spin_unlock_irqrestore(&channel->vc.lock, flags);
461 
462 	list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
463 		kfree(desc);
464 		channel->descs_allocated--;
465 	}
466 
467 	INIT_LIST_HEAD(&channel->ld_free);
468 	vchan_free_chan_resources(&channel->vc);
469 }
470 
471 static struct dma_async_tx_descriptor *
rz_dmac_prep_dma_memcpy(struct dma_chan * chan,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)472 rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
473 			size_t len, unsigned long flags)
474 {
475 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
476 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
477 	struct rz_dmac_desc *desc;
478 
479 	dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
480 		__func__, channel->index, &src, &dest, len);
481 
482 	if (list_empty(&channel->ld_free))
483 		return NULL;
484 
485 	desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
486 
487 	desc->type = RZ_DMAC_DESC_MEMCPY;
488 	desc->src = src;
489 	desc->dest = dest;
490 	desc->len = len;
491 	desc->direction = DMA_MEM_TO_MEM;
492 
493 	list_move_tail(channel->ld_free.next, &channel->ld_queue);
494 	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
495 }
496 
497 static struct dma_async_tx_descriptor *
rz_dmac_prep_slave_sg(struct dma_chan * chan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)498 rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
499 		      unsigned int sg_len,
500 		      enum dma_transfer_direction direction,
501 		      unsigned long flags, void *context)
502 {
503 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
504 	struct rz_dmac_desc *desc;
505 	struct scatterlist *sg;
506 	int dma_length = 0;
507 	int i = 0;
508 
509 	if (list_empty(&channel->ld_free))
510 		return NULL;
511 
512 	desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
513 
514 	for_each_sg(sgl, sg, sg_len, i) {
515 		dma_length += sg_dma_len(sg);
516 	}
517 
518 	desc->type = RZ_DMAC_DESC_SLAVE_SG;
519 	desc->sg = sgl;
520 	desc->sgcount = sg_len;
521 	desc->len = dma_length;
522 	desc->direction = direction;
523 
524 	if (direction == DMA_DEV_TO_MEM)
525 		desc->src = channel->src_per_address;
526 	else
527 		desc->dest = channel->dst_per_address;
528 
529 	list_move_tail(channel->ld_free.next, &channel->ld_queue);
530 	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
531 }
532 
rz_dmac_terminate_all(struct dma_chan * chan)533 static int rz_dmac_terminate_all(struct dma_chan *chan)
534 {
535 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
536 	unsigned long flags;
537 	LIST_HEAD(head);
538 
539 	rz_dmac_disable_hw(channel);
540 	spin_lock_irqsave(&channel->vc.lock, flags);
541 	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
542 	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
543 	vchan_get_all_descriptors(&channel->vc, &head);
544 	spin_unlock_irqrestore(&channel->vc.lock, flags);
545 	vchan_dma_desc_free_list(&channel->vc, &head);
546 
547 	return 0;
548 }
549 
rz_dmac_issue_pending(struct dma_chan * chan)550 static void rz_dmac_issue_pending(struct dma_chan *chan)
551 {
552 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
553 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
554 	struct rz_dmac_desc *desc;
555 	unsigned long flags;
556 
557 	spin_lock_irqsave(&channel->vc.lock, flags);
558 
559 	if (!list_empty(&channel->ld_queue)) {
560 		desc = list_first_entry(&channel->ld_queue,
561 					struct rz_dmac_desc, node);
562 		channel->desc = desc;
563 		if (vchan_issue_pending(&channel->vc)) {
564 			if (rz_dmac_xfer_desc(channel) < 0)
565 				dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
566 					 channel->index);
567 			else
568 				list_move_tail(channel->ld_queue.next,
569 					       &channel->ld_active);
570 		}
571 	}
572 
573 	spin_unlock_irqrestore(&channel->vc.lock, flags);
574 }
575 
rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)576 static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
577 {
578 	u8 i;
579 	static const enum dma_slave_buswidth ds_lut[] = {
580 		DMA_SLAVE_BUSWIDTH_1_BYTE,
581 		DMA_SLAVE_BUSWIDTH_2_BYTES,
582 		DMA_SLAVE_BUSWIDTH_4_BYTES,
583 		DMA_SLAVE_BUSWIDTH_8_BYTES,
584 		DMA_SLAVE_BUSWIDTH_16_BYTES,
585 		DMA_SLAVE_BUSWIDTH_32_BYTES,
586 		DMA_SLAVE_BUSWIDTH_64_BYTES,
587 		DMA_SLAVE_BUSWIDTH_128_BYTES,
588 	};
589 
590 	for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
591 		if (ds_lut[i] == ds)
592 			return i;
593 	}
594 
595 	return CHCFG_DS_INVALID;
596 }
597 
rz_dmac_config(struct dma_chan * chan,struct dma_slave_config * config)598 static int rz_dmac_config(struct dma_chan *chan,
599 			  struct dma_slave_config *config)
600 {
601 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
602 	u32 val;
603 
604 	channel->dst_per_address = config->dst_addr;
605 	channel->chcfg &= ~CHCFG_FILL_DDS_MASK;
606 	if (channel->dst_per_address) {
607 		val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
608 		if (val == CHCFG_DS_INVALID)
609 			return -EINVAL;
610 
611 		channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val);
612 	}
613 
614 	channel->src_per_address = config->src_addr;
615 	channel->chcfg &= ~CHCFG_FILL_SDS_MASK;
616 	if (channel->src_per_address) {
617 		val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
618 		if (val == CHCFG_DS_INVALID)
619 			return -EINVAL;
620 
621 		channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val);
622 	}
623 
624 	return 0;
625 }
626 
rz_dmac_virt_desc_free(struct virt_dma_desc * vd)627 static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
628 {
629 	/*
630 	 * Place holder
631 	 * Descriptor allocation is done during alloc_chan_resources and
632 	 * get freed during free_chan_resources.
633 	 * list is used to manage the descriptors and avoid any memory
634 	 * allocation/free during DMA read/write.
635 	 */
636 }
637 
rz_dmac_device_synchronize(struct dma_chan * chan)638 static void rz_dmac_device_synchronize(struct dma_chan *chan)
639 {
640 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
641 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
642 	u32 chstat;
643 	int ret;
644 
645 	ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
646 				100, 100000, false, channel, CHSTAT, 1);
647 	if (ret < 0)
648 		dev_warn(dmac->dev, "DMA Timeout");
649 
650 	rz_dmac_set_dmars_register(dmac, channel->index, 0);
651 }
652 
653 /*
654  * -----------------------------------------------------------------------------
655  * IRQ handling
656  */
657 
rz_dmac_irq_handle_channel(struct rz_dmac_chan * channel)658 static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
659 {
660 	struct dma_chan *chan = &channel->vc.chan;
661 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
662 	u32 chstat, chctrl;
663 
664 	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
665 	if (chstat & CHSTAT_ER) {
666 		dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
667 			channel->index, chstat);
668 		rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
669 		goto done;
670 	}
671 
672 	chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
673 	rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
674 done:
675 	return;
676 }
677 
rz_dmac_irq_handler(int irq,void * dev_id)678 static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
679 {
680 	struct rz_dmac_chan *channel = dev_id;
681 
682 	if (channel) {
683 		rz_dmac_irq_handle_channel(channel);
684 		return IRQ_WAKE_THREAD;
685 	}
686 	/* handle DMAERR irq */
687 	return IRQ_HANDLED;
688 }
689 
rz_dmac_irq_handler_thread(int irq,void * dev_id)690 static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
691 {
692 	struct rz_dmac_chan *channel = dev_id;
693 	struct rz_dmac_desc *desc = NULL;
694 	unsigned long flags;
695 
696 	spin_lock_irqsave(&channel->vc.lock, flags);
697 
698 	if (list_empty(&channel->ld_active)) {
699 		/* Someone might have called terminate all */
700 		goto out;
701 	}
702 
703 	desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
704 	vchan_cookie_complete(&desc->vd);
705 	list_move_tail(channel->ld_active.next, &channel->ld_free);
706 	if (!list_empty(&channel->ld_queue)) {
707 		desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
708 					node);
709 		channel->desc = desc;
710 		if (rz_dmac_xfer_desc(channel) == 0)
711 			list_move_tail(channel->ld_queue.next, &channel->ld_active);
712 	}
713 out:
714 	spin_unlock_irqrestore(&channel->vc.lock, flags);
715 
716 	return IRQ_HANDLED;
717 }
718 
719 /*
720  * -----------------------------------------------------------------------------
721  * OF xlate and channel filter
722  */
723 
rz_dmac_chan_filter(struct dma_chan * chan,void * arg)724 static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
725 {
726 	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
727 	struct rz_dmac *dmac = to_rz_dmac(chan->device);
728 	struct of_phandle_args *dma_spec = arg;
729 	u32 ch_cfg;
730 
731 	channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
732 	ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
733 	channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
734 			 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
735 
736 	return !test_and_set_bit(channel->mid_rid, dmac->modules);
737 }
738 
rz_dmac_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)739 static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
740 					 struct of_dma *ofdma)
741 {
742 	dma_cap_mask_t mask;
743 
744 	if (dma_spec->args_count != 1)
745 		return NULL;
746 
747 	/* Only slave DMA channels can be allocated via DT */
748 	dma_cap_zero(mask);
749 	dma_cap_set(DMA_SLAVE, mask);
750 
751 	return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec);
752 }
753 
754 /*
755  * -----------------------------------------------------------------------------
756  * Probe and remove
757  */
758 
rz_dmac_chan_probe(struct rz_dmac * dmac,struct rz_dmac_chan * channel,u8 index)759 static int rz_dmac_chan_probe(struct rz_dmac *dmac,
760 			      struct rz_dmac_chan *channel,
761 			      u8 index)
762 {
763 	struct platform_device *pdev = to_platform_device(dmac->dev);
764 	struct rz_lmdesc *lmdesc;
765 	char pdev_irqname[6];
766 	char *irqname;
767 	int ret;
768 
769 	channel->index = index;
770 	channel->mid_rid = -EINVAL;
771 
772 	/* Request the channel interrupt. */
773 	scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
774 	channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
775 	if (channel->irq < 0)
776 		return channel->irq;
777 
778 	irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
779 				 dev_name(dmac->dev), index);
780 	if (!irqname)
781 		return -ENOMEM;
782 
783 	ret = devm_request_threaded_irq(dmac->dev, channel->irq,
784 					rz_dmac_irq_handler,
785 					rz_dmac_irq_handler_thread, 0,
786 					irqname, channel);
787 	if (ret) {
788 		dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
789 			channel->irq, ret);
790 		return ret;
791 	}
792 
793 	/* Set io base address for each channel */
794 	if (index < 8) {
795 		channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
796 			EACH_CHANNEL_OFFSET * index;
797 		channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
798 	} else {
799 		channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
800 			EACH_CHANNEL_OFFSET * (index - 8);
801 		channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
802 	}
803 
804 	/* Allocate descriptors */
805 	lmdesc = dma_alloc_coherent(&pdev->dev,
806 				    sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
807 				    &channel->lmdesc.base_dma, GFP_KERNEL);
808 	if (!lmdesc) {
809 		dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
810 		return -ENOMEM;
811 	}
812 	rz_lmdesc_setup(channel, lmdesc);
813 
814 	/* Initialize register for each channel */
815 	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
816 
817 	channel->vc.desc_free = rz_dmac_virt_desc_free;
818 	vchan_init(&channel->vc, &dmac->engine);
819 	INIT_LIST_HEAD(&channel->ld_queue);
820 	INIT_LIST_HEAD(&channel->ld_free);
821 	INIT_LIST_HEAD(&channel->ld_active);
822 
823 	return 0;
824 }
825 
rz_dmac_parse_of(struct device * dev,struct rz_dmac * dmac)826 static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
827 {
828 	struct device_node *np = dev->of_node;
829 	int ret;
830 
831 	ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
832 	if (ret < 0) {
833 		dev_err(dev, "unable to read dma-channels property\n");
834 		return ret;
835 	}
836 
837 	if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
838 		dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
839 		return -EINVAL;
840 	}
841 
842 	return 0;
843 }
844 
rz_dmac_probe(struct platform_device * pdev)845 static int rz_dmac_probe(struct platform_device *pdev)
846 {
847 	const char *irqname = "error";
848 	struct dma_device *engine;
849 	struct rz_dmac *dmac;
850 	int channel_num;
851 	int ret;
852 	int irq;
853 	u8 i;
854 
855 	dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
856 	if (!dmac)
857 		return -ENOMEM;
858 
859 	dmac->dev = &pdev->dev;
860 	platform_set_drvdata(pdev, dmac);
861 
862 	ret = rz_dmac_parse_of(&pdev->dev, dmac);
863 	if (ret < 0)
864 		return ret;
865 
866 	dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
867 				      sizeof(*dmac->channels), GFP_KERNEL);
868 	if (!dmac->channels)
869 		return -ENOMEM;
870 
871 	/* Request resources */
872 	dmac->base = devm_platform_ioremap_resource(pdev, 0);
873 	if (IS_ERR(dmac->base))
874 		return PTR_ERR(dmac->base);
875 
876 	dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
877 	if (IS_ERR(dmac->ext_base))
878 		return PTR_ERR(dmac->ext_base);
879 
880 	/* Register interrupt handler for error */
881 	irq = platform_get_irq_byname(pdev, irqname);
882 	if (irq < 0)
883 		return irq;
884 
885 	ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
886 			       irqname, NULL);
887 	if (ret) {
888 		dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
889 			irq, ret);
890 		return ret;
891 	}
892 
893 	/* Initialize the channels. */
894 	INIT_LIST_HEAD(&dmac->engine.channels);
895 
896 	dmac->rstc = devm_reset_control_array_get_exclusive(&pdev->dev);
897 	if (IS_ERR(dmac->rstc))
898 		return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),
899 				     "failed to get resets\n");
900 
901 	pm_runtime_enable(&pdev->dev);
902 	ret = pm_runtime_resume_and_get(&pdev->dev);
903 	if (ret < 0) {
904 		dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
905 		goto err_pm_disable;
906 	}
907 
908 	ret = reset_control_deassert(dmac->rstc);
909 	if (ret)
910 		goto err_pm_runtime_put;
911 
912 	for (i = 0; i < dmac->n_channels; i++) {
913 		ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
914 		if (ret < 0)
915 			goto err;
916 	}
917 
918 	/* Register the DMAC as a DMA provider for DT. */
919 	ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
920 					 NULL);
921 	if (ret < 0)
922 		goto err;
923 
924 	/* Register the DMA engine device. */
925 	engine = &dmac->engine;
926 	dma_cap_set(DMA_SLAVE, engine->cap_mask);
927 	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
928 	rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
929 	rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
930 
931 	engine->dev = &pdev->dev;
932 
933 	engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
934 	engine->device_free_chan_resources = rz_dmac_free_chan_resources;
935 	engine->device_tx_status = dma_cookie_status;
936 	engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
937 	engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
938 	engine->device_config = rz_dmac_config;
939 	engine->device_terminate_all = rz_dmac_terminate_all;
940 	engine->device_issue_pending = rz_dmac_issue_pending;
941 	engine->device_synchronize = rz_dmac_device_synchronize;
942 
943 	engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
944 	dma_set_max_seg_size(engine->dev, U32_MAX);
945 
946 	ret = dma_async_device_register(engine);
947 	if (ret < 0) {
948 		dev_err(&pdev->dev, "unable to register\n");
949 		goto dma_register_err;
950 	}
951 	return 0;
952 
953 dma_register_err:
954 	of_dma_controller_free(pdev->dev.of_node);
955 err:
956 	channel_num = i ? i - 1 : 0;
957 	for (i = 0; i < channel_num; i++) {
958 		struct rz_dmac_chan *channel = &dmac->channels[i];
959 
960 		dma_free_coherent(&pdev->dev,
961 				  sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
962 				  channel->lmdesc.base,
963 				  channel->lmdesc.base_dma);
964 	}
965 
966 	reset_control_assert(dmac->rstc);
967 err_pm_runtime_put:
968 	pm_runtime_put(&pdev->dev);
969 err_pm_disable:
970 	pm_runtime_disable(&pdev->dev);
971 
972 	return ret;
973 }
974 
rz_dmac_remove(struct platform_device * pdev)975 static void rz_dmac_remove(struct platform_device *pdev)
976 {
977 	struct rz_dmac *dmac = platform_get_drvdata(pdev);
978 	unsigned int i;
979 
980 	dma_async_device_unregister(&dmac->engine);
981 	of_dma_controller_free(pdev->dev.of_node);
982 	for (i = 0; i < dmac->n_channels; i++) {
983 		struct rz_dmac_chan *channel = &dmac->channels[i];
984 
985 		dma_free_coherent(&pdev->dev,
986 				  sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
987 				  channel->lmdesc.base,
988 				  channel->lmdesc.base_dma);
989 	}
990 	reset_control_assert(dmac->rstc);
991 	pm_runtime_put(&pdev->dev);
992 	pm_runtime_disable(&pdev->dev);
993 }
994 
995 static const struct of_device_id of_rz_dmac_match[] = {
996 	{ .compatible = "renesas,rz-dmac", },
997 	{ /* Sentinel */ }
998 };
999 MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
1000 
1001 static struct platform_driver rz_dmac_driver = {
1002 	.driver		= {
1003 		.name	= "rz-dmac",
1004 		.of_match_table = of_rz_dmac_match,
1005 	},
1006 	.probe		= rz_dmac_probe,
1007 	.remove_new	= rz_dmac_remove,
1008 };
1009 
1010 module_platform_driver(rz_dmac_driver);
1011 
1012 MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
1013 MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
1014 MODULE_LICENSE("GPL v2");
1015